diff --git a/contrib/teamcity/build-configurations.json b/contrib/teamcity/build-configurations.json index 926b23ddc..09d94956f 100644 --- a/contrib/teamcity/build-configurations.json +++ b/contrib/teamcity/build-configurations.json @@ -1,94 +1,100 @@ { "build-asan": { "script": "builds/build-asan.sh", "timeout": 1800 }, "build-autotools": { "script": "builds/build-autotools.sh", "timeout": 1200 }, "build-bench": { "script": "builds/build-bench.sh", "timeout": 1200 }, "build-clang-10": { "script": "builds/build-clang-10.sh", "timeout": 1200 }, "build-clang-tidy": { "script": "builds/build-clang-tidy.sh", "timeout": 600 }, "build-coverage": { "script": "builds/build-coverage.sh", "timeout": 3600 }, "build-diff": { "script": "builds/build-diff.sh", "timeout": 1200 }, "build-ibd": { "script": "builds/build-ibd.sh", "timeout": 14400 }, "build-ibd-no-assumevalid-checkpoint": { "script": "builds/build-ibd-no-assumevalid-checkpoint.sh", "timeout": 21600 }, "build-linux64": { "script": "builds/build-linux64.sh", "timeout": 1200 }, "build-linux-aarch64": { "script": "builds/build-linux-aarch64.sh", - "timeout": 1800 + "timeout": 1800, + "environment": { + "QEMU_LD_PREFIX": "/usr/aarch64-linux-gnu" + } }, "build-linux-arm": { "script": "builds/build-linux-arm.sh", - "timeout": 1800 + "timeout": 1800, + "environment": { + "QEMU_LD_PREFIX": "/usr/arm-linux-gnueabihf" + } }, "build-make-generator": { "script": "builds/build-make-generator.sh", "timeout": 1200 }, "build-master": { "script": "builds/build-master.sh", "timeout": 4800 }, "build-osx": { "script": "builds/build-osx.sh", "timeout": 600 }, "build-secp256k1": { "script": "builds/build-secp256k1.sh", "timeout": 900 }, "build-tsan": { "script": "builds/build-tsan.sh", "timeout": 1800 }, "build-ubsan": { "script": "builds/build-ubsan.sh", "timeout": 1800 }, "build-win64": { "script": "builds/build-win64.sh", "timeout": 1200 }, "build-without-cli": { "script": "builds/build-without-cli.sh", "timeout": 1200 }, "build-without-wallet": { "script": "builds/build-without-wallet.sh", "timeout": 1200 }, "build-without-zmq": { "script": "builds/build-without-zmq.sh", "timeout": 1800 }, "check-seeds": { "script": "builds/check-seeds.sh", "timeout": 600 } } diff --git a/contrib/teamcity/build-configurations.py b/contrib/teamcity/build-configurations.py index 1408ecf99..3b89c7319 100755 --- a/contrib/teamcity/build-configurations.py +++ b/contrib/teamcity/build-configurations.py @@ -1,136 +1,140 @@ #!/usr/bin/env python3 # Copyright (c) 2020 The Bitcoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse import json import os from pathlib import Path, PurePath import signal import subprocess import sys # Default timeout value in seconds. Should be overridden by the # configuration file. DEFAULT_TIMEOUT = 1 * 60 * 60 if sys.version_info < (3, 6): raise SystemError("This script requires python >= 3.6") def main(): script_dir = PurePath(os.path.realpath(__file__)).parent # By default search for a configuration file in the same directory as this # script. default_config_path = Path( script_dir.joinpath("build-configurations.json") ) parser = argparse.ArgumentParser(description="Run a CI build") parser.add_argument( "build", help="The name of the build to run" ) parser.add_argument( "--config", "-c", help="Path to the builds configuration file (default to {})".format( str(default_config_path) ) ) args, unknown_args = parser.parse_known_args() # Check the configuration file exists config_path = Path(args.config) if args.config else default_config_path if not config_path.is_file(): raise FileNotFoundError( "The configuration file does not exist {}".format( str(config_path) ) ) # Read the configuration with open(config_path, encoding="utf-8") as f: config = json.load(f) # Check the target build has an entry in the configuration file build = config.get(args.build, None) if not build: raise AssertionError( "{} is not a valid build identifier. Valid identifiers are {}".format( args.build, list(config.keys()) ) ) # Make sure there is a script file associated with the build... script = build.get("script", None) if script is None: raise AssertionError( "No script provided for the build {}".format( args.build ) ) # ... and that the script file can be executed script_path = Path(script_dir.joinpath(script)) if not script_path.is_file() or not os.access(script_path, os.X_OK): raise FileNotFoundError( "The script file {} does not exist or does not have execution permission".format( str(script_path) ) ) # Find the git root directory git_root = PurePath( subprocess.run( ['git', 'rev-parse', '--show-toplevel'], capture_output=True, check=True, encoding='utf-8', text=True, ).stdout.strip() ) # Create the build directory as needed build_directory = Path(git_root.joinpath(args.build)) build_directory.mkdir(exist_ok=True) # We will provide the required environment variables environment_variables = { "BUILD_DIR": str(build_directory), "CMAKE_PLATFORMS_DIR": git_root.joinpath("cmake", "platforms"), "THREADS": str(os.cpu_count() or 1), "TOPLEVEL": str(git_root), } try: subprocess.run( [str(script_path)] + unknown_args, check=True, cwd=build_directory, - env={**os.environ, **environment_variables}, + env={ + **os.environ, + **environment_variables, + **build.get("environment", {}) + }, timeout=build.get("timeout", DEFAULT_TIMEOUT), ) except subprocess.TimeoutExpired as e: print( "Build {} timed out after {:.1f}s".format( args.build, round(e.timeout, 1) ) ) # Make sure to kill all the child processes, as subprocess only kills # the one we started. It will also kill this python script ! # The return code is 128 + 9 (SIGKILL) = 137. os.killpg(os.getpgid(os.getpid()), signal.SIGKILL) except subprocess.CalledProcessError as e: print( "Build {} failed with exit code {}".format( args.build, e.returncode)) sys.exit(e.returncode) if __name__ == '__main__': main() diff --git a/contrib/teamcity/builds/build-linux-aarch64.sh b/contrib/teamcity/builds/build-linux-aarch64.sh index 1378478d5..7c0d3314a 100755 --- a/contrib/teamcity/builds/build-linux-aarch64.sh +++ b/contrib/teamcity/builds/build-linux-aarch64.sh @@ -1,45 +1,42 @@ #!/usr/bin/env bash export LC_ALL=C.UTF-8 set -euxo pipefail # shellcheck source=../ci-fixture.sh source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" build_static_dependencies CMAKE_FLAGS=( "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_PLATFORMS_DIR}/LinuxAArch64.cmake" # This will prepend our executable commands with the given emulator call "-DCMAKE_CROSSCOMPILING_EMULATOR=$(command -v qemu-aarch64-static)" # The ZMQ functional test will fail with qemu (due to a qemu limitation), # so disable it to avoid the failure. # Extracted from stderr: # Unknown host QEMU_IFLA type: 50 # Unknown host QEMU_IFLA type: 51 # Unknown QEMU_IFLA_BRPORT type 33 "-DBUILD_BITCOIN_ZMQ=OFF" # This is an horrible hack to workaround a qemu bug: # https://bugs.launchpad.net/qemu/+bug/1748612 # Qemu emits a message for unsupported features called by the guest. # Because the output filtering is not working at all, it causes the # qemu stderr to end up in the node stderr and fail the functional # tests. # Disabling the unsupported feature (here bypassing the config # detection) fixes the issue. # FIXME: get rid of the hack, either by using a better qemu version # or by filtering stderr at the framework level. "-DHAVE_DECL_GETIFADDRS=OFF" ) build_with_cmake -# Let qemu know where to find the system libraries -export QEMU_LD_PREFIX=/usr/aarch64-linux-gnu - # Unit tests ninja check ninja check-secp256k1 # Functional tests ninja check-functional diff --git a/contrib/teamcity/builds/build-linux-arm.sh b/contrib/teamcity/builds/build-linux-arm.sh index 22af72818..3a0c1d573 100755 --- a/contrib/teamcity/builds/build-linux-arm.sh +++ b/contrib/teamcity/builds/build-linux-arm.sh @@ -1,45 +1,42 @@ #!/usr/bin/env bash export LC_ALL=C.UTF-8 set -euxo pipefail # shellcheck source=../ci-fixture.sh source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" build_static_dependencies CMAKE_FLAGS=( "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_PLATFORMS_DIR}/LinuxARM.cmake" # This will prepend our executable commands with the given emulator call "-DCMAKE_CROSSCOMPILING_EMULATOR=$(command -v qemu-arm-static)" # The ZMQ functional test will fail with qemu (due to a qemu limitation), # so disable it to avoid the failure. # Extracted from stderr: # Unknown host QEMU_IFLA type: 50 # Unknown host QEMU_IFLA type: 51 # Unknown QEMU_IFLA_BRPORT type 33 "-DBUILD_BITCOIN_ZMQ=OFF" # This is an horrible hack to workaround a qemu bug: # https://bugs.launchpad.net/qemu/+bug/1748612 # Qemu emits a message for unsupported features called by the guest. # Because the output filtering is not working at all, it causes the # qemu stderr to end up in the node stderr and fail the functional # tests. # Disabling the unsupported feature (here bypassing the config # detection) fixes the issue. # FIXME: get rid of the hack, either by using a better qemu version # or by filtering stderr at the framework level. "-DHAVE_DECL_GETIFADDRS=OFF" ) build_with_cmake -# Let qemu know where to find the system libraries -export QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf - # Unit tests ninja check ninja check-secp256k1 # Functional tests ninja check-functional