diff --git a/contrib/teamcity/build-configurations.yml b/contrib/teamcity/build-configurations.yml index 91b2d3c9b..6029d3e9f 100644 --- a/contrib/teamcity/build-configurations.yml +++ b/contrib/teamcity/build-configurations.yml @@ -1,514 +1,521 @@ --- # Templates can be referenced in builds to avoid duplication templates: gitian_builds: script: | "${TOPLEVEL}/contrib/teamcity/gitian.sh" timeout: 7200 artifacts: gitian-results: '' check+secp256k1: targets: - - all - install - install-secp256k1 - - check - check-secp256k1 - check-functional diff-node: runOnDiffRegex: - cmake/ - src/ - test/ - CMakeLists.txt secp256k1: runOnDiffRegex: - src/secp256k1/ Werror: true targets: - - secp256k1 - install-secp256k1 - - check-secp256k1 ibd: targets: - - bitcoind post_build: | "${TOPLEVEL}/contrib/teamcity/ibd.sh" -disablewallet -debug=net timeout: 28800 artifacts: ibd/debug.log: log/debug.log ibd-no-assumevalid-checkpoint: targets: - - bitcoind post_build: | "${TOPLEVEL}/contrib/teamcity/ibd.sh" -disablewallet -assumevalid=0 -checkpoints=0 -debug=net timeout: 28800 artifacts: ibd/debug.log: log/debug.log # The build descriptions. # If a script is defined, then this will be the only step to run. # Otherwise a list of targets can be specified, grouped by parallel runs. # Example: # targets: # - - build11 # - build12 # - - build21 # - build22 # Will run: # ninja build11 build12 # ninja build21 build22 builds: build-asan: Werror: true clang: true fail_fast: true cmake_flags: - '-DCMAKE_CXX_FLAGS=-DARENA_DEBUG' - '-DCRYPTO_USE_ASM=OFF' - '-DENABLE_SANITIZERS=address' templates: - check+secp256k1 timeout: 2400 env: ASAN_OPTIONS: log_path=stdout LSAN_OPTIONS: log_path=stdout build-bench: runOnDiffRegex: - src/bench/ Werror: true cmake_flags: - '-DSECP256K1_ENABLE_MODULE_ECDH=ON' - '-DSECP256K1_ENABLE_MODULE_MULTISET=ON' targets: - - all - install-bitcoin-bench - install-secp256k1-bench - - bench-bitcoin - - bench-secp256k1 timeout: 1200 + artifacts: + src/bench/BitcoinABC_Bench.csv: bench/BitcoinABC_Bench.csv + src/bench/BitcoinABC_Bench.json: bench/BitcoinABC_Bench.json + post_build: | + "${TOPLEVEL}/contrib/teamcity/nanobench_json_to_teamcity_messages.py" \ + "Bitcoin ABC Benchmark" \ + "${BUILD_DIR}/src/bench/BitcoinABC_Bench.json" build-clang: Werror: true clang: true targets: - - all - install - install-secp256k1 - - check - check-secp256k1 templates: - diff-node timeout: 1200 build-clang-tidy: clang: true cmake_flags: - '-DENABLE_CLANG_TIDY=ON' targets: - - all - - check templates: - diff-node timeout: 1800 build-coverage: gcc: true cmake_flags: - '-DENABLE_COVERAGE=ON' - '-DENABLE_BRANCH_COVERAGE=ON' targets: - - coverage-check-extended post_build: | "${TOPLEVEL}/contrib/teamcity/upload-coverage.sh" check-extended timeout: 4800 artifacts: coverage.tar.gz: coverage.tar.gz build-debug: Werror: true cmake_flags: - '-DCMAKE_BUILD_TYPE=Debug' templates: - check+secp256k1 - diff-node timeout: 1200 build-diff: Werror: true targets: - - all - install - install-secp256k1 - - check-all - check-upgrade-activated templates: - diff-node timeout: 1200 build-docs: targets: - - doc-rpc - doc-doxygen post_build: | xvfb-run -a -e /dev/stderr ninja install-manpages-html timeout: 600 artifacts: doc/*: doc build-fuzzer: runOnDiffRegex: - src/test/fuzz/ - test/fuzz/ clang: true Werror: true cmake_flags: - '-DENABLE_SANITIZERS=fuzzer' targets: - - bitcoin-fuzzers build-ibd: templates: - ibd build-ibd-no-assumevalid-checkpoint: templates: - ibd-no-assumevalid-checkpoint build-linux32: cross_build: static_depends: linux32 toolchain: Linux32 templates: - check+secp256k1 timeout: 3600 build-linux64: cross_build: static_depends: linux64 toolchain: Linux64 templates: - check+secp256k1 timeout: 3600 build-linux-aarch64: cross_build: static_depends: linux-aarch64 toolchain: LinuxAArch64 emulator: qemu-aarch64-static cmake_flags: # The ZMQ functional test will fail with qemu (due to a qemu limitation), # so disable it to avoid the failure. # Extracted from stderr: # Unknown host QEMU_IFLA type: 50 # Unknown host QEMU_IFLA type: 51 # Unknown QEMU_IFLA_BRPORT type 33 - "-DBUILD_BITCOIN_ZMQ=OFF" # This is an horrible hack to workaround a qemu bug: # https://bugs.launchpad.net/qemu/+bug/1748612 # Qemu emits a message for unsupported features called by the guest. # Because the output filtering is not working at all, it causes the # qemu stderr to end up in the node stderr and fail the functional # tests. # Disabling the unsupported feature (here bypassing the config # detection) fixes the issue. # FIXME: get rid of the hack, either by using a better qemu version # or by filtering stderr at the framework level. - "-DHAVE_DECL_GETIFADDRS=OFF" templates: - check+secp256k1 timeout: 3600 env: QEMU_LD_PREFIX: /usr/aarch64-linux-gnu build-linux-arm: cross_build: static_depends: linux-arm toolchain: LinuxARM emulator: qemu-arm-static cmake_flags: # The ZMQ functional test will fail with qemu (due to a qemu limitation), # so disable it to avoid the failure. # Extracted from stderr: # Unknown host QEMU_IFLA type: 50 # Unknown host QEMU_IFLA type: 51 # Unknown QEMU_IFLA_BRPORT type 33 - "-DBUILD_BITCOIN_ZMQ=OFF" # This is an horrible hack to workaround a qemu bug: # https://bugs.launchpad.net/qemu/+bug/1748612 # Qemu emits a message for unsupported features called by the guest. # Because the output filtering is not working at all, it causes the # qemu stderr to end up in the node stderr and fail the functional # tests. # Disabling the unsupported feature (here bypassing the config # detection) fixes the issue. # FIXME: get rid of the hack, either by using a better qemu version # or by filtering stderr at the framework level. - "-DHAVE_DECL_GETIFADDRS=OFF" templates: - check+secp256k1 timeout: 3600 env: QEMU_LD_PREFIX: /usr/arm-linux-gnueabihf build-make-generator: generator: name: 'Unix Makefiles' command: make flags: - '-k' templates: - check+secp256k1 timeout: 1200 build-master: Werror: true targets: - - all - install - install-secp256k1 - - check-extended - check-upgrade-activated-extended timeout: 4800 build-native-osx: templates: - check+secp256k1 timeout: 3600 build-osx: cross_build: static_depends: osx toolchain: OSX targets: - - all - install - install-secp256k1 - install-tests post_build: | export PYTHONPATH="${TOPLEVEL}/depends/x86_64-apple-darwin16/native/lib/python3/dist-packages:${PYTHONPATH:-}" ninja osx-dmg timeout: 3600 artifacts: Bitcoin-ABC.dmg: Bitcoin-ABC.dmg build-secp256k1: runOnDiffRegex: - src/secp256k1/ cmake_flags: - '-DSECP256K1_ENABLE_MODULE_ECDH=ON' - '-DSECP256K1_ENABLE_MODULE_MULTISET=ON' templates: - secp256k1 timeout: 600 build-secp256k1-java: runOnDiffRegex: - src/secp256k1/ cmake_flags: - '-DSECP256K1_ENABLE_MODULE_ECDH=ON' - '-DSECP256K1_ENABLE_JNI=ON' - '-DUSE_JEMALLOC=OFF' templates: - secp256k1 timeout: 600 build-tsan: Werror: true clang: true fail_fast: true cmake_flags: - '-DENABLE_SANITIZERS=thread' targets: - - all - install - - check - check-functional timeout: 2400 env: TSAN_OPTIONS: log_path=stdout build-ubsan: Werror: true clang: true fail_fast: true cmake_flags: - '-DENABLE_SANITIZERS=undefined' templates: - check+secp256k1 timeout: 2400 env: UBSAN_OPTIONS: log_path=stdout build-win64: cross_build: static_depends: win64 toolchain: Win64 cmake_flags: - "-DBUILD_BITCOIN_SEEDER=OFF" - "-DCPACK_STRIP_FILES=ON" - "-DUSE_JEMALLOC=OFF" targets: - - all - install - install-secp256k1 - install-tests - - package post_build: | wine "${ARTIFACT_DIR}/bin/test_bitcoin.exe" --run_test=\!radix_tests,rcu_tests timeout: 3600 artifacts: bitcoin-abc-*-x86_64-w64-mingw32.exe: bitcoin-abc-x86_64-w64-mingw32.exe build-without-bip70: Werror: true cmake_flags: - '-DENABLE_BIP70=OFF' targets: - - all - install - - check - check-functional timeout: 1800 build-without-cli: Werror: true cmake_flags: - '-DBUILD_BITCOIN_CLI=OFF' targets: - - all - install - - check-functional timeout: 1200 build-without-qt: Werror: true cmake_flags: - '-DBUILD_BITCOIN_QT=OFF' targets: - - all - install - - check timeout: 1200 build-without-wallet: Werror: true cmake_flags: - '-DBUILD_BITCOIN_WALLET=OFF' targets: - - all - install - - check - check-functional templates: - diff-node timeout: 1200 build-without-zmq: Werror: true cmake_flags: - '-DBUILD_BITCOIN_ZMQ=OFF' targets: - - all - install - - check - check-functional timeout: 1800 check-buildbot: runOnDiffRegex: - contrib/buildbot/ targets: - - check-buildbot timeout: 600 check-seeds: targets: - - bitcoind - bitcoin-cli post_build: | # Run on different ports to avoid a race where the rpc port used in the first run # may not be closed in time for the second to start. SEEDS_DIR="${TOPLEVEL}"/contrib/seeds RPC_PORT=18832 "${SEEDS_DIR}"/check-seeds.sh main 80 RPC_PORT=18833 "${SEEDS_DIR}"/check-seeds.sh test 70 timeout: 600 check-source-control-tools: cmake_flags: - '-DENABLE_SOURCE_CONTROL_TOOLS_TESTS=ON' targets: - - check-source-control-tools timeout: 600 gitian-linux: templates: - gitian_builds env: OS_NAME: linux gitian-osx: templates: - gitian_builds env: OS_NAME: osx gitian-win: templates: - gitian_builds env: OS_NAME: win lint-circular-dependencies: runOnDiffRegex: - src/ - test/lint/ script: | "${TOPLEVEL}/test/lint/lint-circular-dependencies.sh" cashtab-tests: runOnDiffRegex: - web/cashtab/ artifacts: coverage.tar.gz: coverage.tar.gz env: # Any string will work, CI just needs to be defined CI: 'teamcity' JEST_SUITE_NAME: "Cashtab Test suites" JEST_JUNIT_SUITE_NAME: "CashTab Unit Tests" JEST_JUNIT_OUTPUT_DIR: "test/junit" JEST_JUNIT_OUTPUT_NAME: "cashtab.xml" script: | pushd "${TOPLEVEL}/web/cashtab" npm ci npm run build # Make sure the report is available even if the tests fail move_junit_report() { mv test "${BUILD_DIR}/" } trap "move_junit_report" EXIT npm run test --coverage -- \ --reporters=default \ --reporters=jest-junit \ --coverageReporters=text \ --coverageReporters=lcov \ --coverageReporters=teamcity # Generate a summary report lcov \ --rc lcov_branch_coverage=1 \ --summary coverage/lcov.info \ > coverage/lcov-report/coverage-summary.txt 2>&1 pushd coverage/lcov-report tar -czf ../coverage.tar.gz -- * popd mv coverage/coverage.tar.gz "${BUILD_DIR}/" diff --git a/contrib/teamcity/nanobench_json_to_teamcity_messages.py b/contrib/teamcity/nanobench_json_to_teamcity_messages.py new file mode 100755 index 000000000..9d33e8f65 --- /dev/null +++ b/contrib/teamcity/nanobench_json_to_teamcity_messages.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from datetime import timedelta +import json +import sys +from teamcity.messages import TeamcityServiceMessages + +if len(sys.argv) != 3: + print( + """ + Usage: + {} + + Print the teamcity service messages for associating op/s benchmark result + to the tests. + + Requires the teamcity-messages python library: + pip3 install teamcity-messages +""".format(sys.argv[0])) + sys.exit(1) + +suite_name = sys.argv[1] +with open(sys.argv[2], encoding='utf-8') as f: + json_results = json.load(f) + +teamcity_messages = TeamcityServiceMessages() + +teamcity_messages.testSuiteStarted( + suite_name +) + + +def testMeta_number_message(test_name, param_name, param_value): + teamcity_messages.message( + 'testMeta', + type='number', + testName=test_name, + name=param_name, + value='{:.2f}'.format(param_value), + ) + + +for result in json_results.get('results', []): + test_name = result['name'] + + teamcity_messages.testStarted( + test_name + ) + + testMeta_number_message( + test_name, + 'ns/{}'.format(result['unit']), + 1e9 * result['median(elapsed)'] / result['batch'], + ) + + testMeta_number_message( + test_name, + '{}/s'.format(result['unit']), + result['batch'] / result['median(elapsed)'], + ) + + testMeta_number_message( + test_name, + 'err%', + 100 * result['medianAbsolutePercentError(elapsed)'], + ) + + testMeta_number_message( + test_name, + 'ins/{}'.format(result['unit']), + result['median(instructions)'] / result['batch'], + ) + + teamcity_messages.testFinished( + test_name, + testDuration=timedelta(seconds=result['totalTime']), + ) + +teamcity_messages.testSuiteFinished( + suite_name +) diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index 986220cd9..24000295e 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -1,94 +1,101 @@ # Copyright (c) 2018 The Bitcoin developers project(bitcoin-bench) set(BENCH_DATA_RAW_FILES data/block413567.raw ) # Process raw files. file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/data") foreach(_raw_file ${BENCH_DATA_RAW_FILES}) string(APPEND _generated_header_output "${CMAKE_CURRENT_BINARY_DIR}/${_raw_file}" ".h" ) list(APPEND BENCH_DATA_GENERATED_HEADERS ${_generated_header_output}) get_filename_component(_test_name ${_raw_file} NAME_WE) add_custom_command( OUTPUT "${_generated_header_output}" COMMAND "${Python_EXECUTABLE}" "data/convert-raw-to-header.py" "${_test_name}" "${_raw_file}" > "${_generated_header_output}" COMMENT "Transforming raw file ${_raw_file} into header" MAIN_DEPEDENCY "${_raw_file}" DEPENDS "data/convert-raw-to-header.py" VERBATIM WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" ) endforeach() add_executable(bitcoin-bench addrman.cpp base58.cpp bench.cpp bench_bitcoin.cpp block_assemble.cpp cashaddr.cpp ccoins_caching.cpp chacha_poly_aead.cpp chacha20.cpp checkblock.cpp checkqueue.cpp crypto_aes.cpp crypto_hash.cpp data.cpp duplicate_inputs.cpp examples.cpp gcs_filter.cpp hashpadding.cpp lockedpool.cpp mempool_eviction.cpp mempool_stress.cpp merkle_root.cpp nanobench.cpp poly1305.cpp prevector.cpp rollingbloom.cpp rpc_blockchain.cpp rpc_mempool.cpp util_time.cpp verify_script.cpp # Add the generated headers to trigger the conversion command ${BENCH_DATA_GENERATED_HEADERS} ) target_link_libraries(bitcoin-bench testutil) if(BUILD_BITCOIN_WALLET) target_sources(bitcoin-bench PRIVATE coin_selection.cpp wallet_balance.cpp ) target_link_libraries(bitcoin-bench wallet) endif() include(InstallationHelper) install_target(bitcoin-bench EXCLUDE_FROM_ALL) include(TestSuite) add_test_custom_target(bench-bitcoin TEST_COMMAND "$" + -output_csv=BitcoinABC_Bench.csv + -output_json=BitcoinABC_Bench.json CUSTOM_TARGET_ARGS DEPENDS bitcoin-bench USES_TERMINAL ) + +set_property(DIRECTORY "${CMAKE_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_CLEAN_FILES + BitcoinABC_Bench.csv + BitcoinABC_Bench.json +)