diff --git a/cmake/modules/TestSuite.cmake b/cmake/modules/TestSuite.cmake index 4e550cb3f..0729765a7 100644 --- a/cmake/modules/TestSuite.cmake +++ b/cmake/modules/TestSuite.cmake @@ -1,125 +1,159 @@ # Allow to easily build test suites +option(ENABLE_JUNIT_REPORT "Enable Junit report generation for targets that support it" OFF) + +set_property( + DIRECTORY "${CMAKE_SOURCE_DIR}" + APPEND PROPERTY ADDITIONAL_CLEAN_FILES + "${CMAKE_BINARY_DIR}/test/junit" + "${CMAKE_BINARY_DIR}/test/tmp" +) + macro(add_test_environment VARIABLE VALUE) set_property(GLOBAL APPEND PROPERTY TEST_ENVIRONMENT "${VARIABLE}=${VALUE}") endmacro() function(add_test_custom_target TARGET) cmake_parse_arguments(ARG "" "" "CUSTOM_TARGET_ARGS;TEST_COMMAND" ${ARGN}) get_property(TEST_ENVIRONMENT GLOBAL PROPERTY TEST_ENVIRONMENT) add_custom_target(${TARGET} ${ARG_CUSTOM_TARGET_ARGS} COMMAND ${CMAKE_COMMAND} -E env ${TEST_ENVIRONMENT} ${ARG_TEST_COMMAND} ) endfunction() # Define a new target property to hold the list of tests associated with a test # suite. This property is named UNIT_TESTS to avoid confusion with the directory # level property TESTS. define_property(TARGET PROPERTY UNIT_TESTS BRIEF_DOCS "List of tests" FULL_DOCS "A list of the tests associated with a test suite" ) macro(get_target_from_suite SUITE TARGET) set(${TARGET} "check-${SUITE}") endmacro() include(Coverage) function(create_test_suite_with_parent_targets NAME) get_target_from_suite(${NAME} TARGET) add_custom_target(${TARGET} COMMENT "Running ${NAME} test suite" COMMAND cmake -E echo "PASSED: ${NAME} test suite" ) foreach(PARENT_TARGET ${ARGN}) if(TARGET ${PARENT_TARGET}) add_dependencies(${PARENT_TARGET} ${TARGET}) endif() endforeach() add_custom_target_coverage(${TARGET}) endfunction() macro(create_test_suite NAME) create_test_suite_with_parent_targets(${NAME} check-all check-extended) endmacro() set(TEST_RUNNER_TEMPLATE "${CMAKE_CURRENT_LIST_DIR}/../templates/TestRunner.cmake.in") function(add_test_runner SUITE NAME EXECUTABLE) + cmake_parse_arguments(ARG "JUNIT" "" "" ${ARGN}) + get_target_from_suite(${SUITE} SUITE_TARGET) set(TARGET "${SUITE_TARGET}-${NAME}") add_test_custom_target(${TARGET} TEST_COMMAND "${CMAKE_SOURCE_DIR}/cmake/utils/test_wrapper.sh" "${SUITE}-${NAME}.log" - ${CMAKE_CROSSCOMPILING_EMULATOR} "$" ${ARGN} + ${CMAKE_CROSSCOMPILING_EMULATOR} "$" ${ARG_UNPARSED_ARGUMENTS} CUSTOM_TARGET_ARGS COMMENT "${SUITE}: testing ${NAME}" DEPENDS ${EXECUTABLE} VERBATIM ) add_dependencies(${SUITE_TARGET} ${TARGET}) + + if(ENABLE_JUNIT_REPORT AND ARG_JUNIT) + add_custom_command(TARGET ${TARGET} POST_BUILD + COMMENT "Processing junit report for test ${NAME} from suite ${SUITE}" + COMMAND_EXPAND_LISTS + COMMAND + "${Python_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/cmake/utils/junit-reports-merge.py" + "${CMAKE_BINARY_DIR}/test/junit" + "${CMAKE_BINARY_DIR}/test/tmp" + "${SUITE}" + "${NAME}" + ) + endif() endfunction() function(add_test_to_suite SUITE NAME) add_executable(${NAME} EXCLUDE_FROM_ALL ${ARGN}) add_test_runner(${SUITE} ${NAME} ${NAME}) get_target_from_suite(${SUITE} TARGET) set_property( TARGET ${TARGET} APPEND PROPERTY UNIT_TESTS ${NAME} ) endfunction(add_test_to_suite) function(add_boost_unit_tests_to_suite SUITE NAME) cmake_parse_arguments(ARG "" "" "TESTS" ${ARGN} ) get_target_from_suite(${SUITE} SUITE_TARGET) add_executable(${NAME} EXCLUDE_FROM_ALL ${ARG_UNPARSED_ARGUMENTS}) add_dependencies("${SUITE_TARGET}" ${NAME}) + set(HRF_LOGGER "HRF,test_suite") + foreach(_test_source ${ARG_TESTS}) target_sources(${NAME} PRIVATE "${_test_source}") get_filename_component(_test_name "${_test_source}" NAME_WE) + + if(ENABLE_JUNIT_REPORT) + set(JUNIT_LOGGER ":JUNIT,message,${SUITE}-${_test_name}.xml") + endif() + add_test_runner( ${SUITE} ${_test_name} - ${NAME} -t "${_test_name}" -l test_suite + ${NAME} + JUNIT + "--run_test=${_test_name}" + "--logger=${HRF_LOGGER}${JUNIT_LOGGER}" ) set_property( TARGET ${SUITE_TARGET} APPEND PROPERTY UNIT_TESTS ${_test_name} ) endforeach() find_package(Boost 1.59 REQUIRED unit_test_framework) target_link_libraries(${NAME} Boost::unit_test_framework) # We need to detect if the BOOST_TEST_DYN_LINK flag is required include(CheckCXXSourceCompiles) set(CMAKE_REQUIRED_LIBRARIES Boost::unit_test_framework) check_cxx_source_compiles(" #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN #include " BOOST_REQUIRES_TEST_DYN_LINK) if(BOOST_REQUIRES_TEST_DYN_LINK) target_compile_definitions(${NAME} PRIVATE BOOST_TEST_DYN_LINK) endif() endfunction(add_boost_unit_tests_to_suite) diff --git a/cmake/utils/junit-reports-merge.py b/cmake/utils/junit-reports-merge.py new file mode 100755 index 000000000..ed8aefa9b --- /dev/null +++ b/cmake/utils/junit-reports-merge.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import datetime +import fcntl +import os +import sys +import xml.etree.ElementTree as ET + + +class TestSuite: + def __init__(self, name, report_dir): + self.name = name + self.test_cases = {} + self.report_file = os.path.join(report_dir, '{}.xml'.format(self.name)) + + def add_test_case(self, test_case): + self.test_cases[test_case.test_id] = test_case + + def get_failed_tests(self): + return [t for t in self.test_cases.values() if not t.test_success] + + def dump(self): + # Calculate test suite duration as the sum of all test case duraration + duration = round(sum([ + float(t.node.get('time', 0.0)) for t in self.test_cases.values() + ]), 3) + + test_suite = ET.Element( + 'testsuite', + { + 'name': self.name, + 'id': '0', + 'timestamp': datetime.datetime.now().isoformat('T'), + 'time': str(duration), + 'tests': str(len(self.test_cases)), + 'failures': str(len(self.get_failed_tests())), + } + ) + + for test_case in self.test_cases.values(): + test_suite.append(test_case.node) + + report_dir = os.path.dirname(self.report_file) + os.makedirs(report_dir, exist_ok=True) + ET.ElementTree(test_suite).write( + self.report_file, + 'UTF-8', + xml_declaration=True, + ) + + def load(self): + tree = ET.parse(self.report_file) + + xml_root = tree.getroot() + assert xml_root.tag == 'testsuite' + assert self.name == xml_root.get('name') + + for test_case in xml_root.findall('testcase'): + self.add_test_case(TestCase(test_case)) + + +class TestCase: + def __init__(self, node): + self.node = node + self.test_success = self.node.find('failure') is None + + def __getattr__(self, attribute): + if attribute == 'test_id': + return self.classname + '/' + self.name + + return self.node.attrib[attribute] + + +class Lock: + def __init__(self, suite, lock_dir): + self.lock_file = os.path.join(lock_dir, '{}.lock'.format(suite)) + + def __enter__(self): + os.makedirs(os.path.dirname(self.lock_file), exist_ok=True) + self.fd = open(self.lock_file, 'w', encoding='utf-8') + fcntl.lockf(self.fd, fcntl.LOCK_EX) + + def __exit__(self, type, value, traceback): + fcntl.lockf(self.fd, fcntl.LOCK_UN) + self.fd.close() + + +def main(report_dir, lock_dir, suite, test): + junit = '{}-{}.xml'.format(suite, test) + if not os.path.isfile(junit): + return 0 + + tree = ET.parse(junit) + + # Junit root can be a single test suite or multiple test suites. The + # later case is unsupported. + xml_root = tree.getroot() + if xml_root.tag != 'testsuite': + raise AssertionError( + "The parser only supports a single test suite per report") + + test_suite_name = xml_root.get('name') + + lock = Lock(suite, lock_dir) + with lock: + test_suite = TestSuite(test_suite_name, report_dir) + if os.path.isfile(test_suite.report_file): + test_suite.load() + + for child in xml_root: + if child.tag != 'testcase' or (child.find('skipped') is not None): + continue + + test_suite.add_test_case(TestCase(child)) + + test_suite.dump() + + sys.exit( + 1 if test in [case.classname for case in test_suite.get_failed_tests()] + else 0 + ) + + +main( + report_dir=sys.argv[1], + lock_dir=sys.argv[2], + suite=sys.argv[3], + test=sys.argv[4], +) diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index e5a9bd52d..86cfb1f69 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -1,254 +1,267 @@ # Copyright (c) 2018 The Bitcoin developers project(bitcoin-test) option(ENABLE_PROPERTY_BASED_TESTS "Enable property based tests" OFF) # Process json files. file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/data") function(gen_json_header NAME) set(HEADERS "") foreach(f ${ARGN}) set(h "${CMAKE_CURRENT_BINARY_DIR}/${f}.h") # Get the proper name for the test variable. get_filename_component(TEST_NAME ${f} NAME_WE) add_custom_command(OUTPUT ${h} COMMAND "${Python_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/data/generate_header.py" "${TEST_NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/${f}" > ${h} MAIN_DEPENDENCY ${f} DEPENDS "data/generate_header.py" VERBATIM ) list(APPEND HEADERS ${h}) endforeach(f) set(${NAME} "${HEADERS}" PARENT_SCOPE) endfunction() gen_json_header(JSON_HEADERS data/base58_encode_decode.json data/blockfilters.json data/key_io_valid.json data/key_io_invalid.json data/script_tests.json data/sighash.json data/tx_invalid.json data/tx_valid.json ) include(TestSuite) create_test_suite(bitcoin) add_dependencies(check check-bitcoin) # An utility library for bitcoin related test suites. add_library(testutil OBJECT util/blockfilter.cpp util/logging.cpp util/setup_common.cpp util/str.cpp util/transaction_utils.cpp ) target_link_libraries(testutil server) if(BUILD_BITCOIN_WALLET) set(BITCOIN_WALLET_TEST_FIXTURE ../wallet/test/init_test_fixture.cpp ../wallet/test/wallet_test_fixture.cpp ) set(BITCOIN_WALLET_TESTS ../wallet/test/db_tests.cpp ../wallet/test/coinselector_tests.cpp ../wallet/test/init_tests.cpp ../wallet/test/ismine_tests.cpp ../wallet/test/psbt_wallet_tests.cpp ../wallet/test/wallet_tests.cpp ../wallet/test/walletdb_tests.cpp ../wallet/test/wallet_crypto_tests.cpp ) endif() if(ENABLE_PROPERTY_BASED_TESTS) find_package(Rapidcheck REQUIRED) set(BITCOIN_PROPERTY_BASED_TEST_FIXTURE gen/crypto_gen.cpp ) set(BITCOIN_PROPERTY_BASED_TESTS key_properties.cpp ) endif() add_boost_unit_tests_to_suite(bitcoin test_bitcoin fixture.cpp jsonutil.cpp scriptflags.cpp sigutil.cpp # Tests generated from JSON ${JSON_HEADERS} # Wallet test fixture ${BITCOIN_WALLET_TEST_FIXTURE} # Properties based test fixture ${BITCOIN_PROPERTY_BASED_TEST_FIXTURE} TESTS activation_tests.cpp addrman_tests.cpp allocator_tests.cpp amount_tests.cpp arith_uint256_tests.cpp base32_tests.cpp base58_tests.cpp base64_tests.cpp bip32_tests.cpp bitmanip_tests.cpp blockchain_tests.cpp blockcheck_tests.cpp blockencodings_tests.cpp blockfilter_tests.cpp blockfilter_index_tests.cpp blockindex_tests.cpp blockstatus_tests.cpp bloom_tests.cpp bswap_tests.cpp cashaddr_tests.cpp cashaddrenc_tests.cpp checkdatasig_tests.cpp checkpoints_tests.cpp checkqueue_tests.cpp coins_tests.cpp compress_tests.cpp config_tests.cpp core_io_tests.cpp crypto_tests.cpp cuckoocache_tests.cpp dbwrapper_tests.cpp denialofservice_tests.cpp descriptor_tests.cpp dstencode_tests.cpp excessiveblock_tests.cpp feerate_tests.cpp finalization_tests.cpp flatfile_tests.cpp fs_tests.cpp getarg_tests.cpp hash_tests.cpp inv_tests.cpp key_io_tests.cpp key_tests.cpp lcg_tests.cpp limitedmap_tests.cpp mempool_tests.cpp merkle_tests.cpp merkleblock_tests.cpp miner_tests.cpp monolith_opcodes_tests.cpp multisig_tests.cpp net_tests.cpp netbase_tests.cpp op_reversebytes_tests.cpp pmt_tests.cpp policyestimator_tests.cpp prevector_tests.cpp radix_tests.cpp raii_event_tests.cpp random_tests.cpp rcu_tests.cpp reverselock_tests.cpp rpc_tests.cpp rpc_server_tests.cpp rwcollection_tests.cpp sanity_tests.cpp scheduler_tests.cpp schnorr_tests.cpp script_bitfield_tests.cpp script_commitment_tests.cpp script_p2sh_tests.cpp script_standard_tests.cpp script_tests.cpp scriptnum_tests.cpp serialize_tests.cpp settings_tests.cpp sigcache_tests.cpp sigencoding_tests.cpp sighash_tests.cpp sighashtype_tests.cpp sigcheckcount_tests.cpp skiplist_tests.cpp streams_tests.cpp sync_tests.cpp timedata_tests.cpp torcontrol_tests.cpp transaction_tests.cpp txindex_tests.cpp txvalidation_tests.cpp txvalidationcache_tests.cpp uint256_tests.cpp undo_tests.cpp util_tests.cpp util_threadnames_tests.cpp validation_block_tests.cpp validation_tests.cpp validationinterface_tests.cpp versionbits_tests.cpp work_comparator_tests.cpp # RPC Tests ../rpc/test/server_tests.cpp # Wallet tests ${BITCOIN_WALLET_TESTS} # Properties based tests ${BITCOIN_PROPERTY_BASED_TESTS} ) if(ENABLE_PROPERTY_BASED_TESTS) target_link_libraries(test_bitcoin Rapidcheck::rapidcheck) endif() function(add_boost_test_runners_with_upgrade_activated SUITE EXECUTABLE) set(SUITE_UPGRADE_ACTIVATED "${SUITE}-upgrade-activated") get_target_from_suite(${SUITE_UPGRADE_ACTIVATED} TARGET_UPGRADE_ACTIVATED) if(NOT TARGET ${TARGET_UPGRADE_ACTIVATED}) create_test_suite_with_parent_targets( ${SUITE_UPGRADE_ACTIVATED} check-upgrade-activated check-upgrade-activated-extended ) add_dependencies(${TARGET_UPGRADE_ACTIVATED} ${EXECUTABLE}) endif() get_target_from_suite(${SUITE} SUITE_TARGET) get_target_property(BOOST_TESTS ${SUITE_TARGET} UNIT_TESTS) + get_target_from_suite(${SUITE_UPGRADE_ACTIVATED} SUITE_UPGRADE_ACTIVATED_TARGET) + + set(HRF_LOGGER "HRF,test_suite") + foreach(_test_name ${BOOST_TESTS}) + if(ENABLE_JUNIT_REPORT) + set(JUNIT_LOGGER ":JUNIT,message,${SUITE_UPGRADE_ACTIVATED}-${_test_name}.xml") + endif() + add_test_runner( ${SUITE_UPGRADE_ACTIVATED} "${_test_name}" - ${EXECUTABLE} -t "${_test_name}" + ${EXECUTABLE} + JUNIT + "--run_test=${_test_name}" + "--logger=${HRF_LOGGER}${JUNIT_LOGGER}" # Dec. 1st, 2019 at 00:00:00 - -- -axionactivationtime=1575158400 + -- + "-testsuitename=Bitcoin ABC unit tests with next upgrade activated" + -axionactivationtime=1575158400 ) endforeach() endfunction() add_boost_test_runners_with_upgrade_activated(bitcoin test_bitcoin) target_link_libraries(test_bitcoin rpcclient testutil) if(TARGET bitcoinconsensus-shared) target_link_libraries(test_bitcoin bitcoinconsensus-shared) else() target_link_libraries(test_bitcoin bitcoinconsensus) endif() add_subdirectory(fuzz) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e5f4d4f1b..ec414f580 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,134 +1,134 @@ ### # Create config.ini file for tests ### if("fuzzer" IN_LIST ENABLE_SANITIZERS) set(ENABLE_FUZZ ON) else() set(ENABLE_FUZZ OFF) endif() # Create build ini file configure_file(config.ini.cmake.in config.ini) ### # Setup symlinks for testing ### include(SanitizeHelper) function(make_link file) set(src "${CMAKE_CURRENT_SOURCE_DIR}/${file}") set(dest "${CMAKE_CURRENT_BINARY_DIR}/${file}") # Create the target directory and parents if needed. get_filename_component(dest_dir "${dest}" DIRECTORY) file(MAKE_DIRECTORY "${dest_dir}") add_custom_command( OUTPUT "${dest}" COMMAND ${CMAKE_COMMAND} -E create_symlink "${src}" "${dest}" COMMENT "link ${file}" MAIN_DEPENDENCY "${src}" ) # Add a phony target to make sure the files are linked by default. sanitize_target_name("link-" "${file}" NAME) add_custom_target(${NAME} ALL DEPENDS "${dest}") endfunction() make_link(functional/test_runner.py) make_link(util/bitcoin-util-test.py) make_link(util/rpcauth-test.py) make_link(fuzz/test_runner.py) include(Coverage) include(TestSuite) set(_TEST_TARGET_DEPENDS "") if(BUILD_BITCOIN_CLI) list(APPEND _TEST_TARGET_DEPENDS bitcoin-cli) endif() if(BUILD_BITCOIN_WALLET) list(APPEND _TEST_TARGET_DEPENDS bitcoin-wallet) endif() macro(add_functional_test_check TARGET COMMENT) add_test_custom_target(${TARGET} TEST_COMMAND "${Python_EXECUTABLE}" ./functional/test_runner.py ${ARGN} CUSTOM_TARGET_ARGS COMMENT "${COMMENT}" DEPENDS bitcoind ${_TEST_TARGET_DEPENDS} ${CMAKE_CURRENT_BINARY_DIR}/functional/test_runner.py USES_TERMINAL VERBATIM ) add_custom_target_coverage(${TARGET}) endmacro() add_functional_test_check(check-functional "Run the functional tests" ) add_dependencies(check-all check-functional) add_functional_test_check(check-functional-extended "Run the extended functional tests" --extended ) add_dependencies(check-extended check-functional-extended) set(TEST_SUITE_NAME_UPGRADE_ACTIVATED "Bitcoin ABC functional tests with the next upgrade activated") add_functional_test_check(check-functional-upgrade-activated "Run the functional tests with the upgrade activated" --with-axionactivation -n "${TEST_SUITE_NAME_UPGRADE_ACTIVATED}" ) add_dependencies(check-upgrade-activated check-functional-upgrade-activated) add_functional_test_check(check-functional-upgrade-activated-extended "Run the extended functional tests with the upgrade activated" --extended --with-axionactivation -n "${TEST_SUITE_NAME_UPGRADE_ACTIVATED}" ) add_dependencies(check-upgrade-activated-extended check-functional-upgrade-activated-extended) if(BUILD_BITCOIN_TX) add_test_custom_target(check-bitcoin-util TEST_COMMAND "${Python_EXECUTABLE}" ./util/bitcoin-util-test.py CUSTOM_TARGET_ARGS COMMENT "Test Bitcoin utilities..." DEPENDS bitcoin-tx ${CMAKE_CURRENT_BINARY_DIR}/util/bitcoin-util-test.py ) add_dependencies(check check-bitcoin-util) endif() add_custom_target(check-rpcauth COMMENT "Test Bitcoin RPC authentication..." COMMAND "${Python_EXECUTABLE}" ./util/rpcauth-test.py DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/util/rpcauth-test.py ) add_dependencies(check check-rpcauth) include(PackageHelper) exclude_from_source_package( # Subdirectories "cache/" "lint/" "sanitizer_suppressions/" ) -set_property(DIRECTORY "${CMAKE_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_CLEAN_FILES "${CMAKE_CURRENT_BINARY_DIR}/tmp" "${CMAKE_CURRENT_BINARY_DIR}/cache") +set_property(DIRECTORY "${CMAKE_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_CLEAN_FILES "${CMAKE_CURRENT_BINARY_DIR}/cache")