diff --git a/contrib/teamcity/build-configurations.sh b/contrib/teamcity/build-configurations.sh index d370246e02..94c5d5d238 100755 --- a/contrib/teamcity/build-configurations.sh +++ b/contrib/teamcity/build-configurations.sh @@ -1,54 +1,110 @@ #!/usr/bin/env bash export LC_ALL=C set -euxo pipefail if [ -z "$ABC_BUILD_NAME" ]; then echo "Error: Environment variable ABC_BUILD_NAME must be set" exit 1 fi echo "Running build configuration '${ABC_BUILD_NAME}'..." TOPLEVEL=$(git rev-parse --show-toplevel) -cd "${TOPLEVEL}/contrib/teamcity" +export TOPLEVEL + +setup() { + export BUILD_DIR="${TOPLEVEL}/build" + + mkdir -p "${BUILD_DIR}/output" + TEST_RUNNER_FLAGS="--tmpdirprefix=output" + + cd "${BUILD_DIR}" + + # Determine the number of build threads + THREADS=$(nproc || sysctl -n hw.ncpu) + export THREADS + + # Base directories for sanitizer related files + SAN_SUPP_DIR="${TOPLEVEL}/test/sanitizer_suppressions" + SAN_LOG_DIR="${BUILD_DIR}/sanitizer_logs" + + # Create the log directory if it doesn't exist and clear it + mkdir -p "${SAN_LOG_DIR}" + rm -rf "${SAN_LOG_DIR:?}"/* + + # Sanitizers options, not used if sanitizers are not enabled + export ASAN_OPTIONS="malloc_context_size=0:log_path=${SAN_LOG_DIR}/asan.log" + export LSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/lsan:log_path=${SAN_LOG_DIR}/lsan.log" + export TSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/tsan:log_path=${SAN_LOG_DIR}/tsan.log" + export UBSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/ubsan:print_stacktrace=1:halt_on_error=1:log_path=${SAN_LOG_DIR}/ubsan.log" +} + +# Facility to print out sanitizer log outputs to the build log console +print_sanitizers_log() { + for log in "${SAN_LOG_DIR}"/*.log.* + do + echo "*** Output of ${log} ***" + cat "${log}" + done +} + +CI_SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +setup case "$ABC_BUILD_NAME" in build-asan) - export CONFIGURE_FLAGS="--enable-debug --with-sanitizers=address --disable-ccache" - ./build.sh + # Build with the address sanitizer, then run unit tests and functional tests. + CONFIGURE_FLAGS="--enable-debug --with-sanitizers=address --disable-ccache" "${CI_SCRIPTS_DIR}"/build.sh + make -j "${THREADS}" check + ./test/functional/test_runner.py -J=junit_results_asan.xml ${TEST_RUNNER_FLAGS} ;; build-ubsan) - export CONFIGURE_FLAGS="--enable-debug --with-sanitizers=undefined --disable-ccache CC=clang CXX=clang++" - ./build.sh + # Build with the undefined sanitizer, then run unit tests and functional tests. + CONFIGURE_FLAGS="--enable-debug --with-sanitizers=undefined --disable-ccache CC=clang CXX=clang++" "${CI_SCRIPTS_DIR}"/build.sh + make -j "${THREADS}" check + ./test/functional/test_runner.py -J=junit_results_ubsan.xml ${TEST_RUNNER_FLAGS} ;; build-default) - ./build.sh - ./build-secp256k1.sh + # Build, run unit tests and functional tests (with extended cutoff if this is the master branch). + "${CI_SCRIPTS_DIR}"/build.sh + make -j "${THREADS}" check + + BRANCH=$(git rev-parse --abbrev-ref HEAD) + if [[ "${BRANCH}" == "master" ]]; then + TEST_RUNNER_FLAGS="${TEST_RUNNER_FLAGS} --cutoff=600" + fi + ./test/functional/test_runner.py -J=junit_results_default.xml ${TEST_RUNNER_FLAGS} + ./test/functional/test_runner.py -J=junit_results_next_upgrade.xml --with-gravitonactivation ${TEST_RUNNER_FLAGS} + + # Build secp256k1 and run the java tests. + export TOPLEVEL="${TOPLEVEL}"/src/secp256k1 + setup + CONFIGURE_FLAGS="--enable-jni --enable-experimental --enable-module-ecdh" "${CI_SCRIPTS_DIR}"/build.sh + make -j "${THREADS}" check-java ;; build-without-wallet) - export DISABLE_WALLET=1 - ./build.sh + # Build without wallet and run the unit tests. + CONFIGURE_FLAGS="--disable-wallet" "${CI_SCRIPTS_DIR}"/build.sh + make -j "${THREADS}" check ;; build-ibd) - export DISABLE_TESTS=1 - ./build.sh - ./ibd.sh -disablewallet + "${CI_SCRIPTS_DIR}"/build.sh + "${CI_SCRIPTS_DIR}"/ibd.sh -disablewallet ;; build-ibd-no-assumevalid-checkpoint) - export DISABLE_TESTS=1 - ./build.sh - ./ibd.sh -disablewallet -assumevalid=0 -checkpoints=0 + "${CI_SCRIPTS_DIR}"/build.sh + "${CI_SCRIPTS_DIR}"/ibd.sh -disablewallet -assumevalid=0 -checkpoints=0 ;; *) echo "Error: Invalid build name '${ABC_BUILD_NAME}'" exit 2 ;; esac diff --git a/contrib/teamcity/build-secp256k1.sh b/contrib/teamcity/build-secp256k1.sh deleted file mode 100755 index af7a06b9fe..0000000000 --- a/contrib/teamcity/build-secp256k1.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C - -set -euxo pipefail - -TOPLEVEL=$(git rev-parse --show-toplevel) -if [[ -z "${TOPLEVEL}" ]]; then - echo "No .git directory found, assuming pwd" - TOPLEVEL=$(pwd -P) -fi -SECP_DIR="${TOPLEVEL}/src/secp256k1" - -# Generate necessary autoconf files -cd ${SECP_DIR} -./autogen.sh - -# Setup build directory -BUILD_DIR="${SECP_DIR}/build" -mkdir -p ${BUILD_DIR} -cd ${BUILD_DIR} - -# Determine the number of build threads -THREADS=$(nproc || sysctl -n hw.ncpu) - -../configure --enable-jni --enable-experimental --enable-module-ecdh - -# Run build -make -j ${THREADS} - -# Run Java tests -make check-java diff --git a/contrib/teamcity/build.sh b/contrib/teamcity/build.sh index 5dd2cbba2b..613710f469 100755 --- a/contrib/teamcity/build.sh +++ b/contrib/teamcity/build.sh @@ -1,90 +1,25 @@ #!/usr/bin/env bash export LC_ALL=C set -euxo pipefail -TOPLEVEL=$(git rev-parse --show-toplevel) -if [[ -z "${TOPLEVEL}" ]]; then - echo "No .git directory found, assuming pwd" - TOPLEVEL=$(pwd -P) -fi +: "${TOPLEVEL:=$(git rev-parse --show-toplevel)}" +: "${BUILD_DIR:=${TOPLEVEL}/build}" -BUILD_DIR="${TOPLEVEL}/build" -mkdir -p ${BUILD_DIR} +# Default to nothing +: "${CONFIGURE_FLAGS:=}" -## Generate necessary autoconf files +# Generate necessary autoconf files cd ${TOPLEVEL} ./autogen.sh cd ${BUILD_DIR} rm -f build.status test_bitcoin.xml -## Determine the number of build threads -THREADS=$(nproc || sysctl -n hw.ncpu) - -# Default to nothing -: "${DISABLE_WALLET:=}" -: "${CONFIGURE_FLAGS:=}" - read -a CONFIGURE_FLAGS <<< "$CONFIGURE_FLAGS --prefix=$(pwd)" -if [[ -n "${DISABLE_WALLET}" ]]; then - echo "*** Building without wallet" - CONFIGURE_FLAGS+=("--disable-wallet") -fi ../configure "${CONFIGURE_FLAGS[@]}" -# Base directories for sanitizer related files -SAN_SUPP_DIR="${TOPLEVEL}/test/sanitizer_suppressions" -SAN_LOG_DIR="${BUILD_DIR}/sanitizer_logs" - -# Create the log directory if it doesn't exist and clear it -mkdir -p "${SAN_LOG_DIR}" -rm -rf "${SAN_LOG_DIR:?}"/* - -# Sanitizers options, not used if sanitizers are not enabled -export ASAN_OPTIONS="malloc_context_size=0:log_path=${SAN_LOG_DIR}/asan.log" -export LSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/lsan:log_path=${SAN_LOG_DIR}/lsan.log" -export TSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/tsan:log_path=${SAN_LOG_DIR}/tsan.log" -export UBSAN_OPTIONS="suppressions=${SAN_SUPP_DIR}/ubsan:print_stacktrace=1:halt_on_error=1:log_path=${SAN_LOG_DIR}/ubsan.log" - -function print_sanitizers_log() { - for log in "${SAN_LOG_DIR}"/*.log.* - do - echo "*** Output of ${log} ***" - cat "${log}" - done -} - # Run build -make -j ${THREADS} - -# Default to nothing -: "${DISABLE_TESTS:=}" - -# If DISABLE_TESTS is unset (default), run the tests -if [[ -z "${DISABLE_TESTS}" ]]; then - echo "*** Running tests" - - # Run unit tests - make -j ${THREADS} check || (print_sanitizers_log && exit 1) - - mkdir -p output/ - BRANCH=$(git rev-parse --abbrev-ref HEAD) - JUNIT_DEFAULT="junit_results_default.xml" - JUNIT_NEXT_UPGRADE="junit_results_next_upgrade.xml" - - if [[ -n "${DISABLE_WALLET}" ]]; then - echo "Skipping rpc testing due to disabled wallet functionality." - elif [[ "${BRANCH}" == "master" ]]; then - ./test/functional/test_runner.py -J="${JUNIT_DEFAULT}" --cutoff=600 --tmpdirprefix=output - ./test/functional/test_runner.py -J="${JUNIT_NEXT_UPGRADE}" --cutoff=600 --tmpdirprefix=output --with-gravitonactivation - else - ./test/functional/test_runner.py -J="${JUNIT_DEFAULT}" --tmpdirprefix=output - ./test/functional/test_runner.py -J="${JUNIT_NEXT_UPGRADE}" --tmpdirprefix=output --with-gravitonactivation - fi -else - echo "*** Tests have been skipped" -fi - +make -j "${THREADS}" diff --git a/contrib/teamcity/ibd.sh b/contrib/teamcity/ibd.sh index 102791c7b4..3e876ab4e0 100755 --- a/contrib/teamcity/ibd.sh +++ b/contrib/teamcity/ibd.sh @@ -1,41 +1,42 @@ #!/usr/bin/env bash export LC_ALL=C set -euxo pipefail ### # Initial Block Download script. # # Runs a bitcoind process until initial block download is complete. # Forwards the exit code from bitcoind onward. ### MYPID=$$ # Setup -TOPLEVEL=$(git rev-parse --show-toplevel) +: "${TOPLEVEL:=$(git rev-parse --show-toplevel)}" + DATA_DIR="${TOPLEVEL}/ibd" mkdir -p "${DATA_DIR}" DEBUG_LOG="${DATA_DIR}/debug.log" cleanup() { # Cleanup background processes spawned by this script. pkill -P ${MYPID} tail || true } trap "cleanup" EXIT # Make sure the debug log exists so that tail does not fail touch "${DEBUG_LOG}" # Show some progress tail -f "${DEBUG_LOG}" | grep 'UpdateTip' | awk 'NR % 10000 == 0' & callback() { echo "Initial block download complete." # TODO Add more checks to see if IBD completed as expected. # These checks will exit the subshell with a non-zero exit code. } export -f callback LOG_FILE="${DEBUG_LOG}" "${TOPLEVEL}/contrib/devtools/bitcoind-exit-on-log.sh" --grep 'progress=1.000000' --params "-datadir=${DATA_DIR} $*" --callback callback