forked from bitcoin/bitcoin
-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
tests: Add script checking for deterministic line coverage
- Loading branch information
1 parent
1933e38
commit 4320623
Showing
1 changed file
with
151 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| #!/usr/bin/env bash | ||
| # | ||
| # Copyright (c) 2019 The Bitcoin Core developers | ||
| # Distributed under the MIT software license, see the accompanying | ||
| # file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||
| # | ||
| # Test for deterministic coverage across unit test runs. | ||
|
|
||
| export LC_ALL=C | ||
|
|
||
| # Use GCOV_EXECUTABLE="gcov" if compiling with gcc. | ||
| # Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang. | ||
| GCOV_EXECUTABLE="gcov" | ||
|
|
||
| # Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. | ||
| NON_DETERMINISTIC_TESTS=( | ||
| "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2)) | ||
| "denialofservice_tests/DoS_mapOrphans" # denialofservice_tests.cpp: it = mapOrphanTransactions.lower_bound(InsecureRand256()); | ||
| "fs_tests/fsbridge_fstream" # deterministic test failure? | ||
| "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() | ||
| "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() | ||
| "tx_validationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "tx_validationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "wallet_tests/dummy_input_size_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "wallet_tests/importmulti_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "wallet_tests/importwallet_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "wallet_tests/ListCoins" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| "wallet_tests/wallet_disableprivkeys" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) | ||
| ) | ||
|
|
||
| TEST_BITCOIN_BINARY="src/test/test_bitcoin" | ||
|
|
||
| print_usage() { | ||
| echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]" | ||
| } | ||
|
|
||
| N_TEST_RUNS=2 | ||
| BOOST_TEST_RUN_FILTERS="" | ||
| if [[ $# != 0 ]]; then | ||
| if [[ $1 == "--help" ]]; then | ||
| print_usage | ||
| exit | ||
| fi | ||
| PARSED_ARGUMENTS=0 | ||
| if [[ $1 =~ [a-z] ]]; then | ||
| BOOST_TEST_RUN_FILTERS=$1 | ||
| PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) | ||
| shift | ||
| fi | ||
| if [[ $1 =~ ^[0-9]+$ ]]; then | ||
| N_TEST_RUNS=$1 | ||
| PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) | ||
| shift | ||
| fi | ||
| if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then | ||
| print_usage | ||
| exit | ||
| fi | ||
| fi | ||
| if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then | ||
| BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')" | ||
| else | ||
| echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}" | ||
| echo | ||
| fi | ||
|
|
||
| if ! command -v gcov > /dev/null; then | ||
| echo "Error: gcov not installed. Exiting." | ||
| exit 1 | ||
| fi | ||
|
|
||
| if ! command -v gcovr > /dev/null; then | ||
| echo "Error: gcovr not installed. Exiting." | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then | ||
| echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile." | ||
| exit 1 | ||
| fi | ||
|
|
||
| get_file_suffix_count() { | ||
| find src/ -type f -name "*.$1" | wc -l | ||
| } | ||
|
|
||
| if [[ $(get_file_suffix_count gcno) == 0 ]]; then | ||
| echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile." | ||
| exit 1 | ||
| fi | ||
|
|
||
| get_covr_filename() { | ||
| echo "gcovr.run-$1.txt" | ||
| } | ||
|
|
||
| TEST_RUN_ID=0 | ||
| while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do | ||
| TEST_RUN_ID=$((TEST_RUN_ID + 1)) | ||
| echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" | ||
| find src/ -type f -name "*.gcda" -exec rm {} \; | ||
| if [[ $(get_file_suffix_count gcda) != 0 ]]; then | ||
| echo "Error: Stale *.gcda files found. Exiting." | ||
| exit 1 | ||
| fi | ||
| TEST_OUTPUT_TEMPFILE=$(mktemp) | ||
| if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then | ||
| cat "${TEST_OUTPUT_TEMPFILE}" | ||
| rm "${TEST_OUTPUT_TEMPFILE}" | ||
| exit 1 | ||
| fi | ||
| rm "${TEST_OUTPUT_TEMPFILE}" | ||
| if [[ $(get_file_suffix_count gcda) == 0 ]]; then | ||
| echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile." | ||
| exit 1 | ||
| fi | ||
| GCOVR_TEMPFILE=$(mktemp) | ||
| if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then | ||
| echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting." | ||
| exit 1 | ||
| fi | ||
| GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID}) | ||
| mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}" | ||
| if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then | ||
| echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)." | ||
| exit 1 | ||
| fi | ||
| if [[ ${TEST_RUN_ID} != 1 ]]; then | ||
| COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}") | ||
| if [[ ${COVERAGE_DIFF} != "" ]]; then | ||
| echo | ||
| echo "The line coverage is non-deterministic between runs. Exiting." | ||
| echo | ||
| echo "The test suite must be deterministic in the sense that the set of lines executed at least" | ||
| echo "once must be identical between runs. This is a necessary condition for meaningful" | ||
| echo "coverage measuring." | ||
| echo | ||
| echo "${COVERAGE_DIFF}" | ||
| exit 1 | ||
| fi | ||
| rm "${GCOVR_FILENAME}" | ||
| fi | ||
| done | ||
|
|
||
| echo | ||
| echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." | ||
| exit |