diff --git a/contrib/teamcity/build-configurations.json b/contrib/teamcity/build-configurations.json index 09d94956f..fb1ab0474 100644 --- a/contrib/teamcity/build-configurations.json +++ b/contrib/teamcity/build-configurations.json @@ -1,100 +1,104 @@ { - "build-asan": { - "script": "builds/build-asan.sh", - "timeout": 1800 - }, - "build-autotools": { - "script": "builds/build-autotools.sh", - "timeout": 1200 - }, - "build-bench": { - "script": "builds/build-bench.sh", - "timeout": 1200 - }, - "build-clang-10": { - "script": "builds/build-clang-10.sh", - "timeout": 1200 - }, - "build-clang-tidy": { - "script": "builds/build-clang-tidy.sh", - "timeout": 600 - }, - "build-coverage": { - "script": "builds/build-coverage.sh", - "timeout": 3600 - }, - "build-diff": { - "script": "builds/build-diff.sh", - "timeout": 1200 - }, - "build-ibd": { - "script": "builds/build-ibd.sh", - "timeout": 14400 - }, - "build-ibd-no-assumevalid-checkpoint": { - "script": "builds/build-ibd-no-assumevalid-checkpoint.sh", - "timeout": 21600 - }, - "build-linux64": { - "script": "builds/build-linux64.sh", - "timeout": 1200 - }, - "build-linux-aarch64": { - "script": "builds/build-linux-aarch64.sh", - "timeout": 1800, - "environment": { - "QEMU_LD_PREFIX": "/usr/aarch64-linux-gnu" - } - }, - "build-linux-arm": { - "script": "builds/build-linux-arm.sh", - "timeout": 1800, - "environment": { - "QEMU_LD_PREFIX": "/usr/arm-linux-gnueabihf" + "templates": { + }, + "builds": { + "build-asan": { + "script": "builds/build-asan.sh", + "timeout": 1800 + }, + "build-autotools": { + "script": "builds/build-autotools.sh", + "timeout": 1200 + }, + "build-bench": { + "script": "builds/build-bench.sh", + "timeout": 1200 + }, + "build-clang-10": { + "script": "builds/build-clang-10.sh", + "timeout": 1200 + }, + "build-clang-tidy": { + "script": "builds/build-clang-tidy.sh", + "timeout": 600 + }, + "build-coverage": { + "script": "builds/build-coverage.sh", + "timeout": 3600 + }, + "build-diff": { + "script": "builds/build-diff.sh", + "timeout": 1200 + }, + "build-ibd": { + "script": "builds/build-ibd.sh", + "timeout": 14400 + }, + "build-ibd-no-assumevalid-checkpoint": { + "script": "builds/build-ibd-no-assumevalid-checkpoint.sh", + "timeout": 21600 + }, + "build-linux64": { + "script": "builds/build-linux64.sh", + "timeout": 1200 + }, + "build-linux-aarch64": { + "script": "builds/build-linux-aarch64.sh", + "timeout": 1800, + "environment": { + "QEMU_LD_PREFIX": "/usr/aarch64-linux-gnu" + } + }, + "build-linux-arm": { + "script": "builds/build-linux-arm.sh", + "timeout": 1800, + "environment": { + "QEMU_LD_PREFIX": "/usr/arm-linux-gnueabihf" + } + }, + "build-make-generator": { + "script": "builds/build-make-generator.sh", + "timeout": 1200 + }, + "build-master": { + "script": "builds/build-master.sh", + "timeout": 4800 + }, + "build-osx": { + "script": "builds/build-osx.sh", + "timeout": 600 + }, + "build-secp256k1": { + "script": "builds/build-secp256k1.sh", + "timeout": 900 + }, + "build-tsan": { + "script": "builds/build-tsan.sh", + "timeout": 1800 + }, + "build-ubsan": { + "script": "builds/build-ubsan.sh", + "timeout": 1800 + }, + "build-win64": { + "script": "builds/build-win64.sh", + "timeout": 1200 + }, + "build-without-cli": { + "script": "builds/build-without-cli.sh", + "timeout": 1200 + }, + "build-without-wallet": { + "script": "builds/build-without-wallet.sh", + "timeout": 1200 + }, + "build-without-zmq": { + "script": "builds/build-without-zmq.sh", + "timeout": 1800 + }, + "check-seeds": { + "script": "builds/check-seeds.sh", + "timeout": 600 } - }, - "build-make-generator": { - "script": "builds/build-make-generator.sh", - "timeout": 1200 - }, - "build-master": { - "script": "builds/build-master.sh", - "timeout": 4800 - }, - "build-osx": { - "script": "builds/build-osx.sh", - "timeout": 600 - }, - "build-secp256k1": { - "script": "builds/build-secp256k1.sh", - "timeout": 900 - }, - "build-tsan": { - "script": "builds/build-tsan.sh", - "timeout": 1800 - }, - "build-ubsan": { - "script": "builds/build-ubsan.sh", - "timeout": 1800 - }, - "build-win64": { - "script": "builds/build-win64.sh", - "timeout": 1200 - }, - "build-without-cli": { - "script": "builds/build-without-cli.sh", - "timeout": 1200 - }, - "build-without-wallet": { - "script": "builds/build-without-wallet.sh", - "timeout": 1200 - }, - "build-without-zmq": { - "script": "builds/build-without-zmq.sh", - "timeout": 1800 - }, - "check-seeds": { - "script": "builds/check-seeds.sh", - "timeout": 600 } } diff --git a/contrib/teamcity/build-configurations.py b/contrib/teamcity/build-configurations.py index d08e97faa..8b96fb2d3 100755 --- a/contrib/teamcity/build-configurations.py +++ b/contrib/teamcity/build-configurations.py @@ -1,149 +1,178 @@ #!/usr/bin/env python3 # Copyright (c) 2020 The Bitcoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse +from deepmerge import always_merger import json import os from pathlib import Path, PurePath import signal import subprocess import sys from teamcity.messages import TeamcityServiceMessages # Default timeout value in seconds. Should be overridden by the # configuration file. DEFAULT_TIMEOUT = 1 * 60 * 60 if sys.version_info < (3, 6): raise SystemError("This script requires python >= 3.6") def main(): script_dir = PurePath(os.path.realpath(__file__)).parent # By default search for a configuration file in the same directory as this # script. default_config_path = Path( script_dir.joinpath("build-configurations.json") ) parser = argparse.ArgumentParser(description="Run a CI build") parser.add_argument( "build", help="The name of the build to run" ) parser.add_argument( "--config", "-c", help="Path to the builds configuration file (default to {})".format( str(default_config_path) ) ) args, unknown_args = parser.parse_known_args() # Check the configuration file exists config_path = Path(args.config) if args.config else default_config_path if not config_path.is_file(): raise FileNotFoundError( "The configuration file does not exist {}".format( str(config_path) ) ) # Read the configuration with open(config_path, encoding="utf-8") as f: config = json.load(f) + # The configuration root should contain a mandatory element "builds", and it + # should not be empty. + if not config.get("builds", None): + raise AssertionError( + "Invalid configuration file {}: the \"builds\" element is missing or empty".format( + str(config_path) + ) + ) + # Check the target build has an entry in the configuration file - build = config.get(args.build, None) + build = config["builds"].get(args.build, None) if not build: raise AssertionError( "{} is not a valid build identifier. Valid identifiers are {}".format( args.build, list(config.keys()) ) ) + # Get a list of the templates, if any + templates = config.get("templates", {}) + + # If the build references a template, merge the configurations + template_name = build.get("template", None) + if template_name: + # Raise an error if the template does not exist + if template_name not in templates: + raise AssertionError( + "Build {} configuration inherits from template {}, but the template does not exist.".format( + args.build, + template_name + ) + ) + + # The template exists, apply the build configuration on top of the + # template + build = always_merger.merge(templates.get(template_name, {}), build) + # Make sure there is a script file associated with the build... script = build.get("script", None) if script is None: raise AssertionError( "No script provided for the build {}".format( args.build ) ) # ... and that the script file can be executed script_path = Path(script_dir.joinpath(script)) if not script_path.is_file() or not os.access(script_path, os.X_OK): raise FileNotFoundError( "The script file {} does not exist or does not have execution permission".format( str(script_path) ) ) # Find the git root directory git_root = PurePath( subprocess.run( ['git', 'rev-parse', '--show-toplevel'], capture_output=True, check=True, encoding='utf-8', text=True, ).stdout.strip() ) # Create the build directory as needed build_directory = Path(git_root.joinpath('abc-ci-builds', args.build)) build_directory.mkdir(exist_ok=True, parents=True) # We will provide the required environment variables environment_variables = { "BUILD_DIR": str(build_directory), "CMAKE_PLATFORMS_DIR": git_root.joinpath("cmake", "platforms"), "THREADS": str(os.cpu_count() or 1), "TOPLEVEL": str(git_root), } # Let the user know what build is being run. # This makes it easier to retrieve the info from the logs. teamcity_messages = TeamcityServiceMessages() teamcity_messages.customMessage( "Starting build {}".format(args.build), status="NORMAL" ) try: subprocess.run( [str(script_path)] + unknown_args, check=True, cwd=build_directory, env={ **os.environ, **environment_variables, **build.get("environment", {}) }, timeout=build.get("timeout", DEFAULT_TIMEOUT), ) except subprocess.TimeoutExpired as e: print( "Build {} timed out after {:.1f}s".format( args.build, round(e.timeout, 1) ) ) # Make sure to kill all the child processes, as subprocess only kills # the one we started. It will also kill this python script ! # The return code is 128 + 9 (SIGKILL) = 137. os.killpg(os.getpgid(os.getpid()), signal.SIGKILL) except subprocess.CalledProcessError as e: print( "Build {} failed with exit code {}".format( args.build, e.returncode)) sys.exit(e.returncode) if __name__ == '__main__': main() diff --git a/contrib/teamcity/setup-debian-buster.sh b/contrib/teamcity/setup-debian-buster.sh index 18bf69b99..ea4585fd3 100755 --- a/contrib/teamcity/setup-debian-buster.sh +++ b/contrib/teamcity/setup-debian-buster.sh @@ -1,120 +1,122 @@ #!/usr/bin/env bash export LC_ALL=C.UTF-8 set -euxo pipefail dpkg --add-architecture i386 PACKAGES=( arcanist automake autotools-dev binutils bsdmainutils build-essential ccache cmake cppcheck curl flake8 g++-aarch64-linux-gnu g++-arm-linux-gnueabihf git g++-mingw-w64 gnupg gperf imagemagick jq lcov less lib32stdc++-8-dev libboost-all-dev libbz2-dev libc6-dev:i386 libcap-dev libdb++-dev libdb-dev libevent-dev libjemalloc-dev libminiupnpc-dev libprotobuf-dev libqrencode-dev libqt5core5a libqt5dbus5 libqt5gui5 librsvg2-bin libssl-dev libtiff-tools libtinfo5 libtool libzmq3-dev make ninja-build nsis php-codesniffer pkg-config protobuf-compiler python3 python3-autopep8 python3-pip python3-setuptools python3-zmq qemu-user-static qttools5-dev qttools5-dev-tools software-properties-common tar wget wine ) function join_by() { local IFS="$1" shift echo "$*" } apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${PACKAGES[@]}") BACKPORTS=( shellcheck ) echo "deb http://deb.debian.org/debian buster-backports main" | tee -a /etc/apt/sources.list apt-get update DEBIAN_FRONTEND=noninteractive apt-get -t buster-backports install -y $(join_by ' ' "${BACKPORTS[@]}") TEAMCITY_DIR=$(dirname "$0") # FIXME this should no longer be needed starting with Teamcity 2020.1, which # supports Java 11. "${TEAMCITY_DIR}/install_openjdk8.sh" # Install llvm-8 and clang-10 apt-key add "${TEAMCITY_DIR}"/llvm.pub add-apt-repository "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-8 main" add-apt-repository "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-10 main" apt-get update LLVM_PACKAGES=( clang-8 clang-10 clang-format-8 clang-tidy-8 clang-tools-8 ) DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${LLVM_PACKAGES[@]}") update-alternatives --install /usr/bin/clang clang "$(command -v clang-8)" 100 update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-8)" 100 # Use a lower priority to keep clang-8 the default update-alternatives --install /usr/bin/clang clang "$(command -v clang-10)" 50 update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-10)" 50 # Use the mingw posix variant update-alternatives --set x86_64-w64-mingw32-g++ $(command -v x86_64-w64-mingw32-g++-posix) update-alternatives --set x86_64-w64-mingw32-gcc $(command -v x86_64-w64-mingw32-gcc-posix) # Python library for interacting with teamcity pip3 install teamcity-messages +# Python library for merging nested structures +pip3 install deepmerge