diff --git a/contrib/teamcity/build-configurations.sh b/contrib/teamcity/build-configurations.sh index c4b7c31f6..2f299d04b 100755 --- a/contrib/teamcity/build-configurations.sh +++ b/contrib/teamcity/build-configurations.sh @@ -1,299 +1,299 @@ #!/usr/bin/env bash export LC_ALL=C.UTF-8 set -euxo pipefail : "${ABC_BUILD_NAME:=""}" if [ -z "$ABC_BUILD_NAME" ]; then echo "Error: Environment variable ABC_BUILD_NAME must be set" exit 1 fi echo "Running build configuration '${ABC_BUILD_NAME}'..." TOPLEVEL=$(git rev-parse --show-toplevel) export TOPLEVEL setup() { : "${BUILD_DIR:=${TOPLEVEL}/build}" mkdir -p "${BUILD_DIR}" BUILD_DIR=$(cd "${BUILD_DIR}"; pwd) export BUILD_DIR cd "${BUILD_DIR}" # Determine the number of build threads THREADS=$(nproc || sysctl -n hw.ncpu) export THREADS # Base directories for sanitizer related files SAN_SUPP_DIR="${TOPLEVEL}/test/sanitizer_suppressions" SAN_LOG_DIR="/tmp/sanitizer_logs" # Create the log directory if it doesn't exist and clear it mkdir -p "${SAN_LOG_DIR}" rm -rf "${SAN_LOG_DIR:?}"/* # Needed options are set by the build system, add the log path to all runs export ASAN_OPTIONS="log_path=${SAN_LOG_DIR}/asan.log" export LSAN_OPTIONS="log_path=${SAN_LOG_DIR}/lsan.log" export TSAN_OPTIONS="log_path=${SAN_LOG_DIR}/tsan.log" export UBSAN_OPTIONS="log_path=${SAN_LOG_DIR}/ubsan.log" # Unit test logger parameters UNIT_TESTS_JUNIT_LOG_LEVEL=message } run_test_bitcoin() { # Usage: run_test_bitcoin "Context as string" [arguments...] ninja test_bitcoin TEST_BITCOIN_JUNIT="junit_results_unit_tests${1:+_${1// /_}}.xml" TEST_BITCOIN_SUITE_NAME="Bitcoin ABC unit tests${1:+ $1}" # More sanitizer options are needed to run the executable directly ASAN_OPTIONS="malloc_context_size=0:${ASAN_OPTIONS}" \ LSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/lsan:${LSAN_OPTIONS}" \ TSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/tsan:${TSAN_OPTIONS}" \ UBSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/ubsan:print_stacktrace=1:halt_on_error=1:${UBSAN_OPTIONS}" \ ./src/test/test_bitcoin \ --logger=HRF:JUNIT,${UNIT_TESTS_JUNIT_LOG_LEVEL},${TEST_BITCOIN_JUNIT} \ -- \ -testsuitename="${TEST_BITCOIN_SUITE_NAME}" \ "${@:2}" } # Facility to print out sanitizer log outputs to the build log console print_sanitizers_log() { for log in "${SAN_LOG_DIR}"/*.log.* do echo "*** Output of ${log} ***" cat "${log}" done } trap "print_sanitizers_log" ERR CI_SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DEVTOOLS_DIR="${TOPLEVEL}"/contrib/devtools setup case "$ABC_BUILD_NAME" in build-asan) # Build with the address sanitizer, then run unit tests and functional tests. CMAKE_FLAGS=( "-DCMAKE_CXX_FLAGS=-DARENA_DEBUG" "-DCMAKE_BUILD_TYPE=Debug" # ASAN does not support assembly code: https://github.com/google/sanitizers/issues/192 # This will trigger a segfault if the SSE4 implementation is selected for SHA256. # Disabling the assembly works around the issue. "-DCRYPTO_USE_ASM=OFF" "-DENABLE_SANITIZERS=address" "-DCCACHE=OFF" "-DCMAKE_C_COMPILER=clang" "-DCMAKE_CXX_COMPILER=clang++" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh run_test_bitcoin "with address sanitizer" # Libs and utils tests ninja \ check-bitcoin-qt \ check-bitcoin-seeder \ check-bitcoin-util \ ninja check-functional ;; build-ubsan) # Build with the undefined sanitizer, then run unit tests and functional tests. CMAKE_FLAGS=( "-DCMAKE_BUILD_TYPE=Debug" "-DENABLE_SANITIZERS=undefined" "-DCCACHE=OFF" "-DCMAKE_C_COMPILER=clang" "-DCMAKE_CXX_COMPILER=clang++" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh run_test_bitcoin "with undefined sanitizer" # Libs and utils tests ninja \ check-bitcoin-qt \ check-bitcoin-seeder \ check-bitcoin-util \ ninja check-functional ;; build-tsan) # Build with the thread sanitizer, then run unit tests and functional tests. CMAKE_FLAGS=( "-DCMAKE_BUILD_TYPE=Debug" "-DENABLE_SANITIZERS=thread" "-DCCACHE=OFF" "-DCMAKE_C_COMPILER=clang" "-DCMAKE_CXX_COMPILER=clang++" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh run_test_bitcoin "with thread sanitizer" # Libs and utils tests ninja \ check-bitcoin-qt \ check-bitcoin-seeder \ check-bitcoin-util \ ninja check-functional ;; build-diff) # Build, run unit tests and functional tests. CMAKE_FLAGS=( "-DSECP256K1_ENABLE_MODULE_ECDH=ON" "-DSECP256K1_ENABLE_JNI=ON" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh # Unit tests run_test_bitcoin run_test_bitcoin "with next upgrade activated" -phononactivationtime=1575158400 # Libs and tools tests # The leveldb tests need to run alone or they will sometimes fail with # garbage output, see: # https://build.bitcoinabc.org/viewLog.html?buildId=29713&guest=1 ninja check-leveldb ninja \ check-bitcoin-qt \ check-bitcoin-seeder \ check-bitcoin-util \ check-devtools \ check-rpcauth \ check-secp256k1 \ check-univalue \ # Functional tests ninja check-functional ninja check-functional-upgrade-activated ;; build-master) # Build, run unit tests and extended functional tests. CMAKE_FLAGS=( "-DSECP256K1_ENABLE_MODULE_ECDH=ON" "-DSECP256K1_ENABLE_JNI=ON" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh # Unit tests run_test_bitcoin run_test_bitcoin "with next upgrade activated" -phononactivationtime=1575158400 # Libs and tools tests # The leveldb tests need to run alone or they will sometimes fail with # garbage output, see: # https://build.bitcoinabc.org/viewLog.html?buildId=29713&guest=1 ninja check-leveldb ninja \ check-bitcoin-qt \ check-bitcoin-seeder \ check-bitcoin-util \ check-devtools \ check-rpcauth \ check-secp256k1 \ check-univalue \ # Functional tests ninja check-functional-extended ninja check-functional-upgrade-activated-extended ;; build-without-cli) # Build without bitcoin-cli CMAKE_FLAGS=( "-DBUILD_BITCOIN_CLI=OFF" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh ninja check-functional ;; build-without-wallet) # Build without wallet and run the unit tests. CMAKE_FLAGS=( "-DBUILD_BITCOIN_WALLET=OFF" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh ninja check-bitcoin-qt ninja check-functional run_test_bitcoin "without wallet" ;; build-ibd) "${DEVTOOLS_DIR}"/build_cmake.sh "${CI_SCRIPTS_DIR}"/ibd.sh -disablewallet -debug=net ;; build-ibd-no-assumevalid-checkpoint) "${DEVTOOLS_DIR}"/build_cmake.sh "${CI_SCRIPTS_DIR}"/ibd.sh -disablewallet -assumevalid=0 -checkpoints=0 -debug=net ;; build-werror) # Build with variable-length-array and thread-safety-analysis treated as errors CMAKE_FLAGS=( "-DENABLE_WERROR=ON" "-DCMAKE_C_COMPILER=clang" "-DCMAKE_CXX_COMPILER=clang++" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh ;; build-autotools) # Ensure that the build using autotools is not broken "${DEVTOOLS_DIR}"/build_autotools.sh make -j "${THREADS}" check ;; build-bench) # Build and run the benchmarks. CMAKE_FLAGS=( "-DBUILD_BITCOIN_WALLET=ON" "-DSECP256K1_ENABLE_MODULE_ECDH=ON" "-DSECP256K1_ENABLE_MODULE_MULTISET=ON" "-DSECP256K1_ENABLE_MODULE_RECOVERY=ON" ) CMAKE_FLAGS="${CMAKE_FLAGS[*]}" "${DEVTOOLS_DIR}"/build_cmake.sh - ninja bench-bitcoin + ./src/bench/bitcoin-bench -printer=junit > junit_results_bench.xml ninja bench-secp256k1 ;; build-make-generator) # Ensure that the build using cmake and the "Unix Makefiles" generator is # not broken. cd ${BUILD_DIR} git clean -xffd cmake -G "Unix Makefiles" .. make -j "${THREADS}" all check ;; check-seeds) "${DEVTOOLS_DIR}"/build_cmake.sh # Run on different ports to avoid a race where the rpc port used in the # first run may not be closed in time for the second to start. SEEDS_DIR="${TOPLEVEL}"/contrib/seeds RPC_PORT=18832 "${SEEDS_DIR}"/check-seeds.sh main 80 RPC_PORT=18833 "${SEEDS_DIR}"/check-seeds.sh test 70 ;; *) echo "Error: Invalid build name '${ABC_BUILD_NAME}'" exit 2 ;; esac diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index d456be822..b6e08cad6 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,152 +1,185 @@ // Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include void benchmark::ConsolePrinter::header() { std::cout << "# Benchmark, evals, iterations, total, min, max, median" << std::endl; } void benchmark::ConsolePrinter::result(const State &state) { auto results = state.m_elapsed_results; std::sort(results.begin(), results.end()); double total = state.m_num_iters * std::accumulate(results.begin(), results.end(), 0.0); double front = 0; double back = 0; double median = 0; if (!results.empty()) { front = results.front(); back = results.back(); size_t mid = results.size() / 2; median = results[mid]; if (0 == results.size() % 2) { median = (results[mid] + results[mid + 1]) / 2; } } std::cout << std::setprecision(6); std::cout << state.m_name << ", " << state.m_num_evals << ", " << state.m_num_iters << ", " << total << ", " << front << ", " << back << ", " << median << std::endl; } void benchmark::ConsolePrinter::footer() {} benchmark::PlotlyPrinter::PlotlyPrinter(std::string plotly_url, int64_t width, int64_t height) : m_plotly_url(plotly_url), m_width(width), m_height(height) {} void benchmark::PlotlyPrinter::header() { std::cout << "" << "" << "
" << ""; } +void benchmark::JunitPrinter::header() { + std::cout << "" << std::endl; +} + +void benchmark::JunitPrinter::result(const State &state) { + auto results = state.m_elapsed_results; + double bench_duration = + state.m_num_iters * + std::accumulate(results.begin(), results.end(), 0.0); + + /* + * Don't print the results now, we need them all to build the + * node. + */ + bench_results.emplace_back(std::move(state.m_name), bench_duration); + total_duration += bench_duration; +} + +void benchmark::JunitPrinter::footer() { + std::cout << std::setprecision(6); + std::cout << "" << std::endl; + + for (const auto &result : bench_results) { + std::cout << "" << std::endl; + } + + std::cout << "" << std::endl; +} + benchmark::BenchRunner::BenchmarkMap &benchmark::BenchRunner::benchmarks() { static std::map benchmarks_map; return benchmarks_map; } benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func, uint64_t num_iters_for_one_second) { benchmarks().insert( std::make_pair(name, Bench{func, num_iters_for_one_second})); } void benchmark::BenchRunner::RunAll(Printer &printer, uint64_t num_evals, double scaling, const std::string &filter, bool is_list_only) { if (!std::ratio_less_equal::value) { std::cerr << "WARNING: Clock precision is worse than microsecond - " "benchmarks may be less accurate!\n"; } #ifdef DEBUG std::cerr << "WARNING: This is a debug build - may result in slower " "benchmarks.\n"; #endif std::regex reFilter(filter); std::smatch baseMatch; printer.header(); for (const auto &p : benchmarks()) { TestingSetup test{CBaseChainParams::REGTEST}; assert(::ChainActive().Height() == 0); if (!std::regex_match(p.first, baseMatch, reFilter)) { continue; } uint64_t num_iters = static_cast(p.second.num_iters_for_one_second * scaling); if (0 == num_iters) { num_iters = 1; } State state(p.first, num_evals, num_iters, printer); if (!is_list_only) { p.second.func(state); } printer.result(state); } printer.footer(); } bool benchmark::State::UpdateTimer(const benchmark::time_point current_time) { if (m_start_time != time_point()) { std::chrono::duration diff = current_time - m_start_time; m_elapsed_results.push_back(diff.count() / m_num_iters); if (m_elapsed_results.size() == m_num_evals) { return false; } } m_num_iters_left = m_num_iters - 1; return true; } diff --git a/src/bench/bench.h b/src/bench/bench.h index b63f84cab..65c7fcb56 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -1,144 +1,156 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H #include #include #include #include #include #include #include #include // Simple micro-benchmarking framework; API mostly matches a subset of the // Google Benchmark framework (see https://github.com/google/benchmark) // Why not use the Google Benchmark framework? Because adding Yet Another // Dependency (that uses cmake as its build system and has lots of features we // don't need) isn't worth it. /* * Usage: static void CODE_TO_TIME(benchmark::State& state) { ... do any setup needed... while (state.KeepRunning()) { ... do stuff you want to time... } ... do any cleanup needed... } // default to running benchmark for 5000 iterations BENCHMARK(CODE_TO_TIME, 5000); */ namespace benchmark { // In case high_resolution_clock is steady, prefer that, otherwise use // steady_clock. struct best_clock { using hi_res_clock = std::chrono::high_resolution_clock; using steady_clock = std::chrono::steady_clock; using type = std::conditional::type; }; using clock = best_clock::type; using time_point = clock::time_point; using duration = clock::duration; class Printer; class State { public: std::string m_name; uint64_t m_num_iters_left; const uint64_t m_num_iters; const uint64_t m_num_evals; std::vector m_elapsed_results; time_point m_start_time; bool UpdateTimer(time_point finish_time); State(std::string name, uint64_t num_evals, double num_iters, Printer &printer) : m_name(name), m_num_iters_left(0), m_num_iters(num_iters), m_num_evals(num_evals) {} inline bool KeepRunning() { if (m_num_iters_left--) { return true; } bool result = UpdateTimer(clock::now()); // measure again so runtime of UpdateTimer is not included m_start_time = clock::now(); return result; } }; typedef std::function BenchFunction; class BenchRunner { struct Bench { BenchFunction func; uint64_t num_iters_for_one_second; }; typedef std::map BenchmarkMap; static BenchmarkMap &benchmarks(); public: BenchRunner(std::string name, BenchFunction func, uint64_t num_iters_for_one_second); static void RunAll(Printer &printer, uint64_t num_evals, double scaling, const std::string &filter, bool is_list_only); }; // interface to output benchmark results. class Printer { public: virtual ~Printer() {} virtual void header() = 0; virtual void result(const State &state) = 0; virtual void footer() = 0; }; // default printer to console, shows min, max, median. class ConsolePrinter : public Printer { public: void header() override; void result(const State &state) override; void footer() override; }; // creates box plot with plotly.js class PlotlyPrinter : public Printer { public: PlotlyPrinter(std::string plotly_url, int64_t width, int64_t height); void header() override; void result(const State &state) override; void footer() override; private: std::string m_plotly_url; int64_t m_width; int64_t m_height; }; + +// Junit compatible printer, allow to log durations on compatible CI +class JunitPrinter : public Printer { +public: + void header() override; + void result(const State &state) override; + void footer() override; + +private: + std::vector> bench_results; + double total_duration; +}; } // namespace benchmark // BENCHMARK(foo, num_iters_for_one_second) expands to: benchmark::BenchRunner // bench_11foo("foo", num_iterations); // Choose a num_iters_for_one_second that takes roughly 1 second. The goal is // that all benchmarks should take approximately // the same time, and scaling factor can be used that the total time is // appropriate for your system. #define BENCHMARK(n, num_iters_for_one_second) \ benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))( \ BOOST_PP_STRINGIZE(n), n, (num_iters_for_one_second)); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 4260d8a0c..4fa706f7b 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,109 +1,113 @@ // Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include static const int64_t DEFAULT_BENCH_EVALUATIONS = 5; static const char *DEFAULT_BENCH_FILTER = ".*"; static const char *DEFAULT_BENCH_SCALING = "1.0"; static const char *DEFAULT_BENCH_PRINTER = "console"; static const char *DEFAULT_PLOT_PLOTLYURL = "https://cdn.plot.ly/plotly-latest.min.js"; static const int64_t DEFAULT_PLOT_WIDTH = 1024; static const int64_t DEFAULT_PLOT_HEIGHT = 768; static void SetupBenchArgs() { gArgs.AddArg("-?", "Print this help message and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-list", "List benchmarks without executing them. Can be combined " "with -scaling and -filter", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg( "-evals=", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-filter=", strprintf("Regular expression filter to select benchmark by " "name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg( "-scaling=", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg( - "-printer=(console|plot)", + "-printer=(console|junit|plot)", strprintf("Choose printer format. console: print data to console. " - "plot: Print results as HTML graph (default: %s)", + "junit: print results as a Junit compliant XML." + "plot: print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-plot-plotlyurl=", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg( "-plot-width=", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg( "-plot-height=", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); // Hidden gArgs.AddArg("-h", "", ArgsManager::ALLOW_ANY, OptionsCategory::HIDDEN); gArgs.AddArg("-help", "", ArgsManager::ALLOW_ANY, OptionsCategory::HIDDEN); } int main(int argc, char **argv) { SetupBenchArgs(); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error.c_str()); return EXIT_FAILURE; } if (HelpRequested(gArgs)) { std::cout << gArgs.GetHelpMessage(); return EXIT_SUCCESS; } int64_t evaluations = gArgs.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); std::string regex_filter = gArgs.GetArg("-filter", DEFAULT_BENCH_FILTER); std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); bool is_list_only = gArgs.GetBoolArg("-list", false); double scaling_factor; if (!ParseDouble(scaling_str, &scaling_factor)) { tfm::format(std::cerr, "Error parsing scaling factor as double: %s\n", scaling_str.c_str()); return EXIT_FAILURE; } std::unique_ptr printer = std::make_unique(); std::string printer_arg = gArgs.GetArg("-printer", DEFAULT_BENCH_PRINTER); if ("plot" == printer_arg) { printer.reset(new benchmark::PlotlyPrinter( gArgs.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), gArgs.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), gArgs.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); } + if ("junit" == printer_arg) { + printer.reset(new benchmark::JunitPrinter()); + } benchmark::BenchRunner::RunAll(*printer, evaluations, scaling_factor, regex_filter, is_list_only); return EXIT_SUCCESS; }