diff --git a/contrib/debian/test_deterministic_coverage.sh b/contrib/debian/test_deterministic_coverage.sh new file mode 100644 --- /dev/null +++ b/contrib/debian/test_deterministic_coverage.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2021 The Bitcoin ABC developers + +# Files that are known to have a non deterministic coverage. +NON_DETERMINISTIC_FILES=( + blockstatus.h + crypto/sha3.cpp + net_processing.cpp + prevector.h + radix.h + test/blockencodings_tests.cpp + test/denialofservice_tests.cpp + test/prevector_tests.cpp + test/validation_block_tests.cpp + txmempool.cpp + validation.cpp + validationinterface.h +) +EXCLUDE_REGEX="$(IFS="|"; echo "(${NON_DETERMINISTIC_FILES[*]})")" + +print_usage() { + echo + echo "Usage:" + echo " $0 [number of test runs (default: 2)]" + echo + echo "Run this from the build directory, after running \"cmake .. -GNinja -DENABLE_COVERAGE=On\"" + echo "or \"cmake .. -GNinja -DENABLE_COVERAGE=On -DENABLE_BRANCH_COVERAGE=ON\"." + echo +} + +COVERAGE_FILENAME="check-bitcoin.coverage/coverage.txt" +COVERAGE_FILENAME_FIRST_RUN="check-bitcoin.coverage/coverage-run1.txt" + +get_file_suffix_count() { + find src/ -type f -name "*.$1" | wc -l +} + +N_TEST_RUNS=2 +if [[ $# != 0 ]]; then + if [[ $1 == "--help" ]]; then + print_usage + exit + fi + PARSED_ARGUMENTS=0 + if [[ $1 =~ ^[0-9]+$ ]]; then + N_TEST_RUNS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 1 || ${N_TEST_RUNS} -lt 2 ]]; then + print_usage + exit + fi +fi + +if ! command -v gcov > /dev/null; then + echo "Error: gcov not installed. Exiting." + exit 1 +fi + +# First run +echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #1 of ${N_TEST_RUNS}" +ninja > /dev/null +if [[ $(get_file_suffix_count gcno) == 0 ]]; then + echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler." + echo + print_usage + exit 1 +fi +ninja coverage-check-bitcoin > /dev/null +mv ${COVERAGE_FILENAME} ${COVERAGE_FILENAME_FIRST_RUN} + +HEADER=$(head -4 ${COVERAGE_FILENAME_FIRST_RUN} | tail -3) + +# Subsequent runs +TEST_RUN_ID=1 +while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do + TEST_RUN_ID=$((TEST_RUN_ID + 1)) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" + ninja coverage-check-bitcoin > /dev/null + + COVERAGE_DIFF=$(diff ${COVERAGE_FILENAME_FIRST_RUN} ${COVERAGE_FILENAME} | grep -vE "${EXCLUDE_REGEX}" | grep -v "Total:") + if [[ ${COVERAGE_DIFF} != "" ]]; then + echo + echo "The line coverage is non-deterministic between runs. Exiting." + echo + echo "The test suite must be deterministic in the sense that the set of lines executed at least" + echo "once must be identical between runs. This is a necessary condition for meaningful" + echo "coverage measuring." + echo + echo "$HEADER" + echo "${COVERAGE_DIFF}" + exit 1 + fi +done + +echo +echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." +exit