diff --git a/contrib/teamcity/build-configurations.json b/contrib/teamcity/build-configurations.json --- a/contrib/teamcity/build-configurations.json +++ b/contrib/teamcity/build-configurations.json @@ -25,9 +25,17 @@ }, "builds": { "build-asan": { - "script": "builds/build-asan.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "clang": true, + "cmake_flags": [ + "-DCMAKE_CXX_FLAGS=-DARENA_DEBUG", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCRYPTO_USE_ASM=OFF", + "-DENABLE_SANITIZERS=address" + ], + "targets": [ + ["all", "install", "install-secp256k1"], + ["check", "check-secp256k1", "check-functional"] ], "timeout": 1800, "env": { @@ -36,17 +44,28 @@ } }, "build-bench": { - "script": "builds/build-bench.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "cmake_flags": [ + "-DSECP256K1_ENABLE_MODULE_ECDH=ON", + "-DSECP256K1_ENABLE_MODULE_MULTISET=ON" + ], + "targets": [ + ["all", "install-bitcoin-bench", "install-secp256k1-bench"], + ["bench-bitcoin"], + ["bench-secp256k1"] ], "timeout": 1200 }, "build-clang-10": { "runOnDiff": true, - "script": "builds/build-clang-10.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "cmake_flags": [ + "-DCMAKE_C_COMPILER=clang-10", + "-DCMAKE_CXX_COMPILER=clang++-10" + ], + "targets": [ + ["all", "install", "install-secp256k1"], + ["check", "check-secp256k1"] ], "timeout": 1200 }, @@ -73,9 +92,10 @@ }, "build-diff": { "runOnDiff": true, - "script": "builds/build-diff.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "targets": [ + ["all", "install", "install-secp256k1"], + ["check-all", "check-upgrade-activated"] ], "timeout": 1200 }, @@ -151,9 +171,10 @@ "timeout": 1200 }, "build-master": { - "script": "builds/build-master.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "targets": [ + ["all", "install", "install-secp256k1"], + ["check-extended", "check-upgrade-activated-extended"] ], "timeout": 4800 }, @@ -179,9 +200,14 @@ } }, "build-tsan": { - "script": "builds/build-tsan.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "clang": true, + "cmake_flags": [ + "-DENABLE_SANITIZERS=thread" + ], + "targets": [ + ["all", "install"], + ["check", "check-functional"] ], "timeout": 1800, "env": { @@ -189,9 +215,15 @@ } }, "build-ubsan": { - "script": "builds/build-ubsan.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "clang": true, + "cmake_flags": [ + "-DCMAKE_BUILD_TYPE=Debug", + "-DENABLE_SANITIZERS=undefined" + ], + "targets": [ + ["all", "install", "install-secp256k1"], + ["check", "check-secp256k1", "check-functional"] ], "timeout": 1800, "env": { @@ -215,24 +247,36 @@ } }, "build-without-cli": { - "script": "builds/build-without-cli.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "cmake_flags": [ + "-DBUILD_BITCOIN_CLI=OFF" + ], + "targets": [ + ["all", "install"], + ["check-functional"] ], "timeout": 1200 }, "build-without-wallet": { "runOnDiff": true, - "script": "builds/build-without-wallet.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "cmake_flags": [ + "-DBUILD_BITCOIN_WALLET=OFF" + ], + "targets": [ + ["all", "install"], + ["check", "check-functional"] ], "timeout": 1200 }, "build-without-zmq": { - "script": "builds/build-without-zmq.sh", - "templates": [ - "common_unix_artifacts" + "Werror": true, + "cmake_flags": [ + "-DBUILD_BITCOIN_ZMQ=OFF" + ], + "targets": [ + ["all", "install"], + ["check", "check-functional"] ], "timeout": 1800 }, diff --git a/contrib/teamcity/build-configurations.py b/contrib/teamcity/build-configurations.py --- a/contrib/teamcity/build-configurations.py +++ b/contrib/teamcity/build-configurations.py @@ -29,7 +29,8 @@ self.config_file = config_file self.name = None self.config = {} - self.script_path = Path() + self.cmake_flags = [] + self.build_steps = [] self.build_directory = None self.junit_reports_dir = None self.test_logs_dir = None @@ -100,24 +101,6 @@ self.config = always_merger.merge(template_config, build) - # Make sure there is a script file associated with the build... - script = self.config.get("script", None) - if script is None: - raise AssertionError( - "No script provided for the build {}".format( - self.name - ) - ) - - # ... and that the script file can be executed - self.script_path = Path(self.script_root.joinpath(script)) - if not self.script_path.is_file() or not os.access(self.script_path, os.X_OK): - raise FileNotFoundError( - "The script file {} does not exist or does not have execution permission".format( - str(self.script_path) - ) - ) - # Create the build directory as needed self.build_directory = Path( self.project_root.joinpath( @@ -129,6 +112,76 @@ self.junit_reports_dir = self.build_directory.joinpath("test/junit") self.test_logs_dir = self.build_directory.joinpath("test/log") + def create_build_steps(self, artifact_dir): + # There are 2 possibilities to define the build steps: + # - By defining a script to run. If such a script is set and is + # executable, it is the only thing to run. + # - By defining the configuration options and a list of target groups to + # run. The configuration step should be run once then all the targets + # groups. Each target group can contain 1 or more targets which + # should be run parallel. + script = self.config.get("script", None) + if script: + script_path = Path(self.script_root.joinpath(script)) + if not script_path.is_file() or not os.access(script_path, os.X_OK): + raise FileNotFoundError( + "The script file {} does not exist or does not have execution permission".format( + str(script_path) + ) + ) + self.build_steps = [ + { + "bin": str(script_path), + "args": [], + } + ] + return + + # Get the cmake configuration definitions. + self.cmake_flags = self.config.get("cmake_flags", []) + self.cmake_flags.append("-DCMAKE_INSTALL_PREFIX={}".format( + str(artifact_dir))) + # Get the targets to build. If none is provided then raise an error. + targets = self.config.get("targets", None) + if not targets: + raise AssertionError( + "No build target has been provided for build {} and no script is defined, aborting".format( + self.name + ) + ) + + # Some more flags for the build_cmake.sh script + build_cmake_flags = [] + if self.config.get("Werror", False): + build_cmake_flags.append("--Werror") + if self.config.get("junit", True): + build_cmake_flags.append("--junit") + if self.config.get("clang", False): + build_cmake_flags.append("--clang") + + # Some generator flags + generator_flags = [] + if self.config.get("fail_fast", False): + generator_flags.append("-k0") + + # First call should use the build_cmake.sh script in order to run + # cmake. + self.build_steps = [ + { + "bin": str(self.project_root.joinpath("contrib/devtools/build_cmake.sh")), + "args": targets[0] + build_cmake_flags, + } + ] + + for target_group in targets[1:]: + self.build_steps.append( + { + # TODO: let the generator be configurable + "bin": "ninja", + "args": generator_flags + target_group, + } + ) + def get(self, key, default): return self.config.get(key, default) @@ -229,9 +282,9 @@ ) continue - async def run_build(self, args=[]): - proc = await asyncio.create_subprocess_exec( - *([str(self.configuration.script_path)] + args), + def run_process(self, bin, args=[]): + return asyncio.create_subprocess_exec( + *([bin] + args), # Buffer limit is 64KB by default, but we need a larger buffer: limit=1024 * 256, stdout=asyncio.subprocess.PIPE, @@ -240,10 +293,14 @@ env={ **os.environ, **self.environment_variables, - **self.configuration.get("env", {}) + **self.configuration.get("env", {}), + "CMAKE_FLAGS": " ".join(self.configuration.cmake_flags), }, ) + async def run_build(self, bin, args=[]): + proc = await self.run_process(bin, args) + await asyncio.wait([ self.process_stdout(proc.stdout) ]) @@ -255,12 +312,15 @@ self.configuration.name ) try: - return_code = await asyncio.wait_for(self.run_build(args), timeout) - if return_code != 0: - message = "Build {} failed with exit code {}".format( - self.configuration.name, - return_code - ) + for step in self.configuration.build_steps: + return_code = await asyncio.wait_for(self.run_build(step["bin"], step["args"]), timeout) + if return_code != 0: + message = "Build {} failed with exit code {}".format( + self.configuration.name, + return_code + ) + return + except asyncio.TimeoutError: message = "Build {} timed out after {:.1f}s".format( self.configuration.name, round(timeout, 1) @@ -290,6 +350,8 @@ shutil.rmtree(self.artifact_dir) self.artifact_dir.mkdir(exist_ok=True) + self.configuration.create_build_steps(self.artifact_dir) + return_code, message = asyncio.run( self.wait_for_build( self.configuration.get( diff --git a/contrib/teamcity/builds/build-asan.sh b/contrib/teamcity/builds/build-asan.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-asan.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build with the address sanitizer, then run unit tests and functional tests. -CMAKE_FLAGS=( - "-DCMAKE_CXX_FLAGS=-DARENA_DEBUG" - "-DCMAKE_BUILD_TYPE=Debug" - # ASAN does not support assembly code: https://github.com/google/sanitizers/issues/192 - # This will trigger a segfault if the SSE4 implementation is selected for SHA256. - # Disabling the assembly works around the issue. - "-DCRYPTO_USE_ASM=OFF" - "-DENABLE_SANITIZERS=address" -) -build_with_cmake --Werror --clang --junit - -ninja -k0 check check-secp256k1 check-functional diff --git a/contrib/teamcity/builds/build-bench.sh b/contrib/teamcity/builds/build-bench.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-bench.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build and run the benchmarks. -CMAKE_FLAGS=( - "-DBUILD_BITCOIN_WALLET=ON" - "-DSECP256K1_ENABLE_MODULE_ECDH=ON" - "-DSECP256K1_ENABLE_MODULE_MULTISET=ON" -) -build_with_cmake --Werror --junit - -ninja bench-bitcoin -ninja bench-secp256k1 diff --git a/contrib/teamcity/builds/build-clang-10.sh b/contrib/teamcity/builds/build-clang-10.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-clang-10.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Use clang-10 for this build instead of the default clang-8. -# This allow for checking that no warning is introduced for newer versions -# of the compiler -CMAKE_FLAGS=( - "-DCMAKE_C_COMPILER=clang-10" - "-DCMAKE_CXX_COMPILER=clang++-10" -) -build_with_cmake --Werror --junit - -ninja -k0 check check-secp256k1 - -# TODO do the same with the latest GCC diff --git a/contrib/teamcity/builds/build-diff.sh b/contrib/teamcity/builds/build-diff.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-diff.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build, run unit tests and functional tests. -build_with_cmake --Werror --junit - -# Libs and tools tests -# The leveldb tests need to run alone or they will sometimes fail with -# garbage output, see: -# https://build.bitcoinabc.org/viewLog.html?buildId=29713&guest=1 -ninja check-leveldb - -ninja -k0 \ - check \ - check-secp256k1 \ - check-univalue \ - check-functional \ - check-upgrade-activated diff --git a/contrib/teamcity/builds/build-master.sh b/contrib/teamcity/builds/build-master.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-master.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build, run unit tests and extended functional tests. -build_with_cmake --Werror --junit - -# Libs and tools tests -# The leveldb tests need to run alone or they will sometimes fail with -# garbage output, see: -# https://build.bitcoinabc.org/viewLog.html?buildId=29713&guest=1 -ninja check-leveldb - -ninja -k0 \ - check \ - check-secp256k1 \ - check-univalue \ - check-functional-extended \ - check-upgrade-activated-extended diff --git a/contrib/teamcity/builds/build-tsan.sh b/contrib/teamcity/builds/build-tsan.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-tsan.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build with the thread sanitizer, then run unit tests and functional tests. -CMAKE_FLAGS=( - "-DENABLE_SANITIZERS=thread" -) -build_with_cmake --Werror --clang --junit - -# Libs and utils tests -ninja -k0 check check-functional diff --git a/contrib/teamcity/builds/build-ubsan.sh b/contrib/teamcity/builds/build-ubsan.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-ubsan.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build with the undefined sanitizer, then run unit tests and functional tests. -CMAKE_FLAGS=( - "-DCMAKE_BUILD_TYPE=Debug" - "-DENABLE_SANITIZERS=undefined" -) -build_with_cmake --Werror --clang --junit - -# Libs and utils tests -ninja -k0 check check-secp256k1 check-functional diff --git a/contrib/teamcity/builds/build-without-cli.sh b/contrib/teamcity/builds/build-without-cli.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-without-cli.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build without bitcoin-cli -CMAKE_FLAGS=( - "-DBUILD_BITCOIN_CLI=OFF" -) -build_with_cmake --Werror --junit - -ninja check-functional diff --git a/contrib/teamcity/builds/build-without-wallet.sh b/contrib/teamcity/builds/build-without-wallet.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-without-wallet.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build without wallet and run the unit tests. -CMAKE_FLAGS=( - "-DBUILD_BITCOIN_WALLET=OFF" -) -build_with_cmake --Werror --junit - -ninja -k0 check check-functional diff --git a/contrib/teamcity/builds/build-without-zmq.sh b/contrib/teamcity/builds/build-without-zmq.sh deleted file mode 100755 --- a/contrib/teamcity/builds/build-without-zmq.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C.UTF-8 - -set -euxo pipefail - -# shellcheck source=../ci-fixture.sh -source "${TOPLEVEL}/contrib/teamcity/ci-fixture.sh" - -# Build without Zeromq and run the unit tests. -CMAKE_FLAGS=( - "-DBUILD_BITCOIN_ZMQ=OFF" -) -build_with_cmake --Werror --junit - -ninja -k0 check check-functional