diff --git a/contrib/teamcity/build-configurations.json b/contrib/teamcity/build-configurations.json --- a/contrib/teamcity/build-configurations.json +++ b/contrib/teamcity/build-configurations.json @@ -1,49 +1,98 @@ { "templates": { + "common_unix_artifacts": { + "artifacts": { + "CMakeCache.txt": "CMakeCache.txt", + "src/bitcoind": "bin/bitcoind", + "src/bitcoin-*": "bin", + "src/qt/bitcoin-qt": "bin/bitcoin-qt", + "src/bench/bitcoin-bench": "bin/bitcoin-bench", + "src/seeder/bitcoin-seeder": "bin/bitcoin-seeder", + "src/libbitcoinconsensus.*": "lib", + "src/test/test_bitcoin": "bin/test_bitcoin", + "src/qt/test/test_bitcoin-qt": "bin/test_bitcoin-qt", + "src/seeder/test/test_bitcoin-seeder": "bin/test_bitcoin-seeder", + "src/qt/test/test_bitcoin-qt.log": "log/qt/test_bitcoin-qt.log", + "src/seeder/test/*.log": "log/seeder", + "test/tmp/*": "functional" + } + } }, "builds": { "build-asan": { "script": "builds/build-asan.sh", - "timeout": 1800 + "template": "common_unix_artifacts", + "timeout": 1800, + "artifacts": { + "sanitizer_logs": "log" + } }, "build-autotools": { "script": "builds/build-autotools.sh", - "timeout": 1200 + "template": "common_unix_artifacts", + "timeout": 1200, + "artifacts": { + "src/bench/bench_bitcoin": "bin/bench_bitcoin", + "config.log": "log/config.log", + "src/test-suite.log": "log/test-suite.log", + "src/test/*.log": "log", + "src/seeder/test/test_bitcoin-seeder.log": "log/test_bitcoin-seeder.log" + } }, "build-bench": { "script": "builds/build-bench.sh", + "template": "common_unix_artifacts", "timeout": 1200 }, "build-clang-10": { "script": "builds/build-clang-10.sh", + "template": "common_unix_artifacts", "timeout": 1200 }, "build-clang-tidy": { "script": "builds/build-clang-tidy.sh", - "timeout": 600 + "template": "common_unix_artifacts", + "timeout": 600, + "artifacts": { + "clang-tidy-warnings.txt": "clang-tidy-warnings.txt" + } }, "build-coverage": { "script": "builds/build-coverage.sh", - "timeout": 3600 + "template": "common_unix_artifacts", + "timeout": 3600, + "artifacts": { + "coverage.tar.gz": "coverage.tar.gz" + } }, "build-diff": { "script": "builds/build-diff.sh", + "template": "common_unix_artifacts", "timeout": 1200 }, "build-ibd": { "script": "builds/build-ibd.sh", - "timeout": 14400 + "template": "common_unix_artifacts", + "timeout": 14400, + "artifacts": { + "ibd/debug.log": "log/debug.log" + } }, "build-ibd-no-assumevalid-checkpoint": { "script": "builds/build-ibd-no-assumevalid-checkpoint.sh", - "timeout": 21600 + "timeout": 21600, + "artifacts": { + "ibd/debug.log": "log/debug.log" + } }, "build-linux64": { "script": "builds/build-linux64.sh", + "template": "common_unix_artifacts", "timeout": 1200 }, "build-linux-aarch64": { "script": "builds/build-linux-aarch64.sh", + "template": "common_unix_artifacts", "timeout": 1800, "environment": { "QEMU_LD_PREFIX": "/usr/aarch64-linux-gnu" @@ -51,6 +100,7 @@ }, "build-linux-arm": { "script": "builds/build-linux-arm.sh", + "template": "common_unix_artifacts", "timeout": 1800, "environment": { "QEMU_LD_PREFIX": "/usr/arm-linux-gnueabihf" @@ -58,46 +108,81 @@ }, "build-make-generator": { "script": "builds/build-make-generator.sh", + "template": "common_unix_artifacts", "timeout": 1200 }, "build-master": { "script": "builds/build-master.sh", + "template": "common_unix_artifacts", "timeout": 4800 }, "build-osx": { "script": "builds/build-osx.sh", - "timeout": 600 + "template": "common_unix_artifacts", + "timeout": 600, + "artifacts": { + "src/qt/BitcoinABC-Qt.app": "bin", + "Bitcoin-ABC.dmg": "Bitcoin-ABC.dmg" + } }, "build-secp256k1": { "script": "builds/build-secp256k1.sh", - "timeout": 900 + "template": "common_unix_artifacts", + "timeout": 900, + "artifacts": { + "src/secp256k1/libsecp256k1*": "lib" + } }, "build-tsan": { "script": "builds/build-tsan.sh", - "timeout": 1800 + "template": "common_unix_artifacts", + "timeout": 1800, + "artifacts": { + "sanitizer_logs": "log" + } }, "build-ubsan": { "script": "builds/build-ubsan.sh", - "timeout": 1800 + "template": "common_unix_artifacts", + "timeout": 1800, + "artifacts": { + "sanitizer_logs": "log" + } }, "build-win64": { "script": "builds/build-win64.sh", - "timeout": 1200 + "timeout": 1200, + "artifacts": { + "CMakeCache.txt": "CMakeCache.txt", + "src/bitcoind.exe": "bin/bitcoind.exe", + "src/bitcoin-*.exe": "bin", + "src/qt/bitcoin-qt.exe": "bin/bitcoin-qt.exe", + "src/bench/bitcoin-bench.exe": "bin/bitcoin-bench.exe", + "src/libbitcoinconsensus*": "lib", + "src/test/test_bitcoin.exe": "bin/test_bitcoin.exe", + "src/qt/test/test_bitcoin-qt.exe": "bin/test_bitcoin-qt.exe", + "src/qt/test/test_bitcoin-qt.log": "log/qt/test_bitcoin-qt.log", + "bitcoin-abc-*-x86_64-w64-mingw32.exe": "bitcoin-abc-x86_64-w64-mingw32.exe" + } }, "build-without-cli": { "script": "builds/build-without-cli.sh", + "template": "common_unix_artifacts", "timeout": 1200 }, "build-without-wallet": { "script": "builds/build-without-wallet.sh", + "template": "common_unix_artifacts", "timeout": 1200 }, "build-without-zmq": { "script": "builds/build-without-zmq.sh", + "template": "common_unix_artifacts", "timeout": 1800 }, "check-seeds": { "script": "builds/check-seeds.sh", + "template": "common_unix_artifacts", "timeout": 600 } } diff --git a/contrib/teamcity/build-configurations.py b/contrib/teamcity/build-configurations.py --- a/contrib/teamcity/build-configurations.py +++ b/contrib/teamcity/build-configurations.py @@ -9,8 +9,10 @@ import os from pathlib import Path, PurePath import signal +import shutil import subprocess import sys +from teamcity import is_running_under_teamcity from teamcity.messages import TeamcityServiceMessages # Default timeout value in seconds. Should be overridden by the @@ -21,6 +23,59 @@ raise SystemError("This script requires python >= 3.6") +def copy_artifacts(teamcity_messages, build_dir, artifacts): + # This accounts for the volume mapping from the container. + # Our local /result is mapped to some relative ./results on the host, so we + # use /results/artifacts to copy our files but results/artifacts as an + # artifact path for teamcity. + # TODO abstract out the volume mapping + if is_running_under_teamcity(): + artifact_dir = Path("/results/artifacts") + else: + artifact_dir = build_dir.joinpath("artifacts") + + if artifact_dir.is_dir(): + shutil.rmtree(artifact_dir) + artifact_dir.mkdir(exist_ok=True) + + # Find and copy artifacts. + # The source is relative to the build tree, the destination relative to the + # artifact directory. + # The artifact directory is located in the build directory tree, results + # from it needs to be excluded from the glob matches to prevent infinite + # recursion. + for pattern, dest in artifacts.items(): + matches = [m for m in sorted(build_dir.glob( + pattern)) if artifact_dir not in m.parents and artifact_dir != m] + dest = artifact_dir.joinpath(dest) + + # Pattern did not match + if not matches: + continue + + # If there is a single file, destination is the new file path + if len(matches) == 1 and matches[0].is_file(): + # Create the parent directories as needed + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(matches[0], dest) + continue + + # If there are multiple files or a single directory, destination is a + # directory. + dest.mkdir(parents=True, exist_ok=True) + for match in matches: + if match.is_file(): + shutil.copy2(match, dest) + else: + shutil.copytree(match, dest.joinpath(match.name)) + + # Instruct teamcity to upload our artifact directory + artifact_path_pattern = "+:{}=>artifacts.tar.gz".format( + str(artifact_dir.relative_to("/")) + ) + teamcity_messages.publishArtifacts(artifact_path_pattern) + + def main(): script_dir = PurePath(os.path.realpath(__file__)).parent @@ -144,6 +199,9 @@ status="NORMAL" ) + # Flag to indicate that the process and its children should be killed + kill_em_all = False + try: subprocess.run( [str(script_path)] + unknown_args, @@ -165,13 +223,23 @@ # Make sure to kill all the child processes, as subprocess only kills # the one we started. It will also kill this python script ! # The return code is 128 + 9 (SIGKILL) = 137. - os.killpg(os.getpgid(os.getpid()), signal.SIGKILL) + kill_em_all = True except subprocess.CalledProcessError as e: print( "Build {} failed with exit code {}".format( args.build, e.returncode)) sys.exit(e.returncode) + finally: + copy_artifacts( + teamcity_messages, + build_directory, + build.get("artifacts", {}) + ) + + # Seek and destroy + if kill_em_all: + os.killpg(os.getpgid(os.getpid()), signal.SIGKILL) if __name__ == '__main__': diff --git a/contrib/teamcity/ci-fixture.sh b/contrib/teamcity/ci-fixture.sh --- a/contrib/teamcity/ci-fixture.sh +++ b/contrib/teamcity/ci-fixture.sh @@ -5,12 +5,13 @@ set -euxo pipefail : "${TOPLEVEL:=$(git rev-parse --show-toplevel)}" +: "${BUILD_DIR:=${TOPLEVEL}/build}" DEVTOOLS_DIR="${TOPLEVEL}/contrib/devtools" # Base directories for sanitizer related files SAN_SUPP_DIR="${TOPLEVEL}/test/sanitizer_suppressions" -SAN_LOG_DIR="/tmp/sanitizer_logs" +SAN_LOG_DIR="${BUILD_DIR}/sanitizer_logs" # Create the log directory if it doesn't exist and clear it mkdir -p "${SAN_LOG_DIR}" diff --git a/contrib/teamcity/ibd.sh b/contrib/teamcity/ibd.sh --- a/contrib/teamcity/ibd.sh +++ b/contrib/teamcity/ibd.sh @@ -18,7 +18,7 @@ : "${BUILD_DIR:=${TOPLEVEL}/build}" : "${BITCOIND:=${BUILD_DIR}/src/bitcoind}" -DATA_DIR="${TOPLEVEL}/ibd" +DATA_DIR="${BUILD_DIR}/ibd" mkdir -p "${DATA_DIR}" DEBUG_LOG="${DATA_DIR}/debug.log"