diff --git a/contrib/buildbot/requirements.txt b/contrib/buildbot/requirements.txt --- a/contrib/buildbot/requirements.txt +++ b/contrib/buildbot/requirements.txt @@ -1,5 +1,6 @@ mock flask phabricator==0.7.0 +pyyaml requests slackclient diff --git a/contrib/buildbot/server.py b/contrib/buildbot/server.py --- a/contrib/buildbot/server.py +++ b/contrib/buildbot/server.py @@ -10,7 +10,6 @@ from functools import wraps import hashlib import hmac -import json import os from phabricator_wrapper import ( BITCOIN_ABC_PROJECT_PHID, @@ -19,6 +18,7 @@ from shieldio import RasterBadge from shlex import quote from teamcity import TeamcityRequestException +import yaml # Some keywords used by TeamCity and tcWebHook @@ -218,8 +218,8 @@ target_phid = get_mandatory_argument('targetPHID') # Get the configuration from master - config = json.loads(phab.get_file_content_from_master( - "contrib/teamcity/build-configurations.json")) + config = yaml.safe_load(phab.get_file_content_from_master( + "contrib/teamcity/build-configurations.yaml")) # Get a list of the builds that should run on diffs builds = [ @@ -449,7 +449,7 @@ ).format(project_name) # secp256k1 is a special case because it has a Travis build from a - # Github repo that is not managed by the build-configurations.json config. + # Github repo that is not managed by the build-configurations.yml config. # The status always need to be fetched. sepc256k1_default_branch = 'master' sepc256k1_travis_status = travis.get_branch_status( @@ -473,8 +473,8 @@ panel_content = add_line_to_panel('') # Download the build configuration from master - config = json.loads(phab.get_file_content_from_master( - "contrib/teamcity/build-configurations.json")) + config = yaml.safe_load(phab.get_file_content_from_master( + "contrib/teamcity/build-configurations.yml")) # Get a list of the builds to display config_build_names = [ diff --git a/contrib/teamcity/build-configurations.json b/contrib/teamcity/build-configurations.json deleted file mode 100644 --- a/contrib/teamcity/build-configurations.json +++ /dev/null @@ -1,324 +0,0 @@ -{ - "templates": { - "common_unix_artifacts": { - "artifacts": { - "CMakeCache.txt": "CMakeCache.txt", - "src/bitcoind": "bin/bitcoind", - "src/bitcoin-*": "bin", - "src/qt/bitcoin-qt": "bin/bitcoin-qt", - "src/bench/bitcoin-bench": "bin/bitcoin-bench", - "src/seeder/bitcoin-seeder": "bin/bitcoin-seeder", - "src/libbitcoinconsensus.*": "lib", - "src/test/test_bitcoin": "bin/test_bitcoin", - "src/qt/test/test_bitcoin-qt": "bin/test_bitcoin-qt", - "src/seeder/test/test-seeder": "bin/test-seeder", - "test/tmp/test_runner_*": "functional" - } - }, - "gitian_builds": { - "script": "gitian.sh", - "timeout": 7200, - "artifacts": { - "gitian-results": "" - } - } - }, - "builds": { - "build-asan": { - "Werror": true, - "clang": true, - "cmake_flags": [ - "-DCMAKE_CXX_FLAGS=-DARENA_DEBUG", - "-DCMAKE_BUILD_TYPE=Debug", - "-DCRYPTO_USE_ASM=OFF", - "-DENABLE_SANITIZERS=address" - ], - "targets": [ - ["all", "install", "install-secp256k1"], - ["check", "check-secp256k1", "check-functional"] - ], - "timeout": 1800, - "env": { - "ASAN_OPTIONS": "log_path=stdout", - "LSAN_OPTIONS": "log_path=stdout" - } - }, - "build-bench": { - "Werror": true, - "cmake_flags": [ - "-DSECP256K1_ENABLE_MODULE_ECDH=ON", - "-DSECP256K1_ENABLE_MODULE_MULTISET=ON" - ], - "targets": [ - ["all", "install-bitcoin-bench", "install-secp256k1-bench"], - ["bench-bitcoin"], - ["bench-secp256k1"] - ], - "timeout": 1200 - }, - "build-clang-10": { - "runOnDiff": true, - "Werror": true, - "cmake_flags": [ - "-DCMAKE_C_COMPILER=clang-10", - "-DCMAKE_CXX_COMPILER=clang++-10" - ], - "targets": [ - ["all", "install", "install-secp256k1"], - ["check", "check-secp256k1"] - ], - "timeout": 1200 - }, - "build-clang-tidy": { - "runOnDiff": true, - "script": "builds/build-clang-tidy.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 600, - "artifacts": { - "clang-tidy-warnings.txt": "clang-tidy-warnings.txt" - } - }, - "build-coverage": { - "script": "builds/build-coverage.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 4800, - "artifacts": { - "coverage.tar.gz": "coverage.tar.gz" - } - }, - "build-diff": { - "runOnDiff": true, - "Werror": true, - "targets": [ - ["all", "install", "install-secp256k1"], - ["check-all", "check-upgrade-activated"] - ], - "timeout": 1200 - }, - "build-docs": { - "script": "builds/build-docs.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 600, - "artifacts": { - "doc/*": "doc" - } - }, - "build-ibd": { - "script": "builds/build-ibd.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 14400, - "artifacts": { - "ibd/debug.log": "log/debug.log" - } - }, - "build-ibd-no-assumevalid-checkpoint": { - "script": "builds/build-ibd-no-assumevalid-checkpoint.sh", - "template": [ - "common_unix_artifacts" - ], - "timeout": 21600, - "artifacts": { - "ibd/debug.log": "log/debug.log" - } - }, - "build-linux32": { - "script": "builds/build-linux32.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 3600 - }, - "build-linux64": { - "script": "builds/build-linux64.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 3600 - }, - "build-linux-aarch64": { - "script": "builds/build-linux-aarch64.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 3600, - "env": { - "QEMU_LD_PREFIX": "/usr/aarch64-linux-gnu" - } - }, - "build-linux-arm": { - "script": "builds/build-linux-arm.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 3600, - "env": { - "QEMU_LD_PREFIX": "/usr/arm-linux-gnueabihf" - } - }, - "build-make-generator": { - "script": "builds/build-make-generator.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 1200 - }, - "build-master": { - "Werror": true, - "targets": [ - ["all", "install", "install-secp256k1"], - ["check-extended", "check-upgrade-activated-extended"] - ], - "timeout": 4800 - }, - "build-osx": { - "script": "builds/build-osx.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 3600, - "artifacts": { - "src/qt/BitcoinABC-Qt.app": "bin", - "Bitcoin-ABC.dmg": "Bitcoin-ABC.dmg" - } - }, - "build-secp256k1": { - "script": "builds/build-secp256k1.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 900, - "artifacts": { - "src/secp256k1/libsecp256k1*": "lib" - } - }, - "build-tsan": { - "Werror": true, - "clang": true, - "cmake_flags": [ - "-DENABLE_SANITIZERS=thread" - ], - "targets": [ - ["all", "install"], - ["check", "check-functional"] - ], - "timeout": 1800, - "env": { - "TSAN_OPTIONS": "log_path=stdout" - } - }, - "build-ubsan": { - "Werror": true, - "clang": true, - "cmake_flags": [ - "-DCMAKE_BUILD_TYPE=Debug", - "-DENABLE_SANITIZERS=undefined" - ], - "targets": [ - ["all", "install", "install-secp256k1"], - ["check", "check-secp256k1", "check-functional"] - ], - "timeout": 1800, - "env": { - "UBSAN_OPTIONS": "log_path=stdout" - } - }, - "build-win64": { - "script": "builds/build-win64.sh", - "timeout": 3600, - "artifacts": { - "CMakeCache.txt": "CMakeCache.txt", - "src/bitcoind.exe": "bin/bitcoind.exe", - "src/bitcoin-*.exe": "bin", - "src/qt/bitcoin-qt.exe": "bin/bitcoin-qt.exe", - "src/bench/bitcoin-bench.exe": "bin/bitcoin-bench.exe", - "src/libbitcoinconsensus*": "lib", - "src/test/test_bitcoin.exe": "bin/test_bitcoin.exe", - "src/qt/test/test_bitcoin-qt.exe": "bin/test_bitcoin-qt.exe", - "src/qt/test/test_bitcoin-qt.log": "log/qt/test_bitcoin-qt.log", - "bitcoin-abc-*-x86_64-w64-mingw32.exe": "bitcoin-abc-x86_64-w64-mingw32.exe" - } - }, - "build-without-cli": { - "Werror": true, - "cmake_flags": [ - "-DBUILD_BITCOIN_CLI=OFF" - ], - "targets": [ - ["all", "install"], - ["check-functional"] - ], - "timeout": 1200 - }, - "build-without-wallet": { - "runOnDiff": true, - "Werror": true, - "cmake_flags": [ - "-DBUILD_BITCOIN_WALLET=OFF" - ], - "targets": [ - ["all", "install"], - ["check", "check-functional"] - ], - "timeout": 1200 - }, - "build-without-zmq": { - "Werror": true, - "cmake_flags": [ - "-DBUILD_BITCOIN_ZMQ=OFF" - ], - "targets": [ - ["all", "install"], - ["check", "check-functional"] - ], - "timeout": 1800 - }, - "check-seeds": { - "script": "builds/check-seeds.sh", - "templates": [ - "common_unix_artifacts" - ], - "timeout": 600 - }, - "check-source-control-tools": { - "cmake_flags": [ - "-DBUILD_SOURCE_CONTROL_TOOLS=ON" - ], - "targets": [ - ["check-source-control-tools"] - ], - "timeout": 600 - }, - "gitian-linux": { - "templates": [ - "gitian_builds" - ], - "env": { - "OS_NAME": "linux" - } - }, - "gitian-osx": { - "templates": [ - "gitian_builds" - ], - "env": { - "OS_NAME": "osx" - } - }, - "gitian-win": { - "templates": [ - "gitian_builds" - ], - "env": { - "OS_NAME": "win" - } - } - } -} diff --git a/contrib/teamcity/build-configurations.py b/contrib/teamcity/build-configurations.py --- a/contrib/teamcity/build-configurations.py +++ b/contrib/teamcity/build-configurations.py @@ -6,7 +6,6 @@ import argparse import asyncio from deepmerge import always_merger -import json import os from pathlib import Path, PurePath import shutil @@ -14,6 +13,7 @@ import sys from teamcity import is_running_under_teamcity from teamcity.messages import TeamcityServiceMessages +import yaml # Default timeout value in seconds. Should be overridden by the # configuration file. @@ -60,7 +60,7 @@ # Read the configuration with open(self.config_file, encoding="utf-8") as f: - config = json.load(f) + config = yaml.safe_load(f) # The configuration root should contain a mandatory element "builds", and # it should not be empty. @@ -439,7 +439,7 @@ # By default search for a configuration file in the same directory as this # script. default_config_path = Path( - script_dir.joinpath("build-configurations.json") + script_dir.joinpath("build-configurations.yml") ) parser = argparse.ArgumentParser(description="Run a CI build") diff --git a/contrib/teamcity/build-configurations.yml b/contrib/teamcity/build-configurations.yml new file mode 100644 --- /dev/null +++ b/contrib/teamcity/build-configurations.yml @@ -0,0 +1,305 @@ +--- +# Templates can be referenced in builds to avoid duplication +templates: + common_unix_artifacts: + artifacts: + CMakeCache.txt: CMakeCache.txt + src/bitcoind: bin/bitcoind + src/bitcoin-*: bin + src/qt/bitcoin-qt: bin/bitcoin-qt + src/bench/bitcoin-bench: bin/bitcoin-bench + src/seeder/bitcoin-seeder: bin/bitcoin-seeder + src/libbitcoinconsensus.*: lib + src/test/test_bitcoin: bin/test_bitcoin + src/qt/test/test_bitcoin-qt: bin/test_bitcoin-qt + src/seeder/test/test-seeder: bin/test-seeder + test/tmp/test_runner_*: functional + + gitian_builds: + script: gitian.sh + timeout: 7200 + artifacts: + gitian-results: '' + +# The build descriptions. +# If a script is defined, then this will be the only step to run. +# Otherwise a list of targets can be specified, grouped by parallel runs. +# Example: +# targets: +# - - build11 +# - build12 +# - - build21 +# - build22 +# Will run: +# ninja build11 build12 +# ninja build21 build22 +builds: + build-asan: + Werror: true + clang: true + cmake_flags: + - '-DCMAKE_CXX_FLAGS=-DARENA_DEBUG' + - '-DCMAKE_BUILD_TYPE=Debug' + - '-DCRYPTO_USE_ASM=OFF' + - '-DENABLE_SANITIZERS=address' + targets: + - - all + - install + - install-secp256k1 + - - check + - check-secp256k1 + - check-functional + timeout: 1800 + env: + ASAN_OPTIONS: log_path=stdout + LSAN_OPTIONS: log_path=stdout + + build-bench: + Werror: true + cmake_flags: + - '-DSECP256K1_ENABLE_MODULE_ECDH=ON' + - '-DSECP256K1_ENABLE_MODULE_MULTISET=ON' + targets: + - - all + - install-bitcoin-bench + - install-secp256k1-bench + - - bench-bitcoin + - - bench-secp256k1 + timeout: 1200 + + build-clang-10: + runOnDiff: true + Werror: true + cmake_flags: + - '-DCMAKE_C_COMPILER=clang-10' + - '-DCMAKE_CXX_COMPILER=clang++-10' + targets: + - - all + - install + - install-secp256k1 + - - check + - check-secp256k1 + timeout: 1200 + + build-clang-tidy: + runOnDiff: true + script: builds/build-clang-tidy.sh + templates: + - common_unix_artifacts + timeout: 600 + artifacts: + clang-tidy-warnings.txt: clang-tidy-warnings.txt + + build-coverage: + script: builds/build-coverage.sh + templates: + - common_unix_artifacts + timeout: 4800 + artifacts: + coverage.tar.gz: coverage.tar.gz + + build-diff: + runOnDiff: true + Werror: true + targets: + - - all + - install + - install-secp256k1 + - - check-all + - check-upgrade-activated + timeout: 1200 + + build-docs: + script: builds/build-docs.sh + templates: + - common_unix_artifacts + timeout: 600 + artifacts: + doc/*: doc + + build-ibd: + script: builds/build-ibd.sh + templates: + - common_unix_artifacts + timeout: 14400 + artifacts: + ibd/debug.log: log/debug.log + + build-ibd-no-assumevalid-checkpoint: + script: builds/build-ibd-no-assumevalid-checkpoint.sh + template: + - common_unix_artifacts + timeout: 21600 + artifacts: + ibd/debug.log: log/debug.log + + build-linux32: + script: builds/build-linux32.sh + templates: + - common_unix_artifacts + timeout: 3600 + + build-linux64: + script: builds/build-linux64.sh + templates: + - common_unix_artifacts + timeout: 3600 + + build-linux-aarch64: + script: builds/build-linux-aarch64.sh + templates: + - common_unix_artifacts + timeout: 3600 + env: + QEMU_LD_PREFIX: /usr/aarch64-linux-gnu + + build-linux-arm: + script: builds/build-linux-arm.sh + templates: + - common_unix_artifacts + timeout: 3600 + env: + QEMU_LD_PREFIX: /usr/arm-linux-gnueabihf + + build-make-generator: + script: builds/build-make-generator.sh + templates: + - common_unix_artifacts + timeout: 1200 + + build-master: + Werror: true + targets: + - - all + - install + - install-secp256k1 + - - check-extended + - check-upgrade-activated-extended + timeout: 4800 + + build-osx: + script: builds/build-osx.sh + templates: + - common_unix_artifacts + timeout: 3600 + artifacts: + src/qt/BitcoinABC-Qt.app: bin + Bitcoin-ABC.dmg: Bitcoin-ABC.dmg + + build-secp256k1: + script: builds/build-secp256k1.sh + templates: + - common_unix_artifacts + timeout: 900 + artifacts: + src/secp256k1/libsecp256k1*: lib + + build-tsan: + Werror: true + clang: true + cmake_flags: + - '-DENABLE_SANITIZERS=thread' + targets: + - - all + - install + - - check + - check-functional + timeout: 1800 + env: + TSAN_OPTIONS: log_path=stdout + + build-ubsan: + Werror: true + clang: true + cmake_flags: + - '-DCMAKE_BUILD_TYPE=Debug' + - '-DENABLE_SANITIZERS=undefined' + targets: + - - all + - install + - install-secp256k1 + - - check + - check-secp256k1 + - check-functional + timeout: 1800 + env: + UBSAN_OPTIONS: log_path=stdout + + build-win64: + script: builds/build-win64.sh + timeout: 3600 + artifacts: + CMakeCache.txt: CMakeCache.txt + src/bitcoind.exe: bin/bitcoind.exe + src/bitcoin-*.exe: bin + src/qt/bitcoin-qt.exe: bin/bitcoin-qt.exe + src/bench/bitcoin-bench.exe: bin/bitcoin-bench.exe + src/libbitcoinconsensus*: lib + src/test/test_bitcoin.exe: bin/test_bitcoin.exe + src/qt/test/test_bitcoin-qt.exe: bin/test_bitcoin-qt.exe + src/qt/test/test_bitcoin-qt.log: log/qt/test_bitcoin-qt.log + bitcoin-abc-*-x86_64-w64-mingw32.exe: bitcoin-abc-x86_64-w64-mingw32.exe + + build-without-cli: + Werror: true + cmake_flags: + - '-DBUILD_BITCOIN_CLI=OFF' + targets: + - - all + - install + - - check-functional + timeout: 1200 + + build-without-wallet: + runOnDiff: true + Werror: true + cmake_flags: + - '-DBUILD_BITCOIN_WALLET=OFF' + targets: + - - all + - install + - - check + - check-functional + timeout: 1200 + + build-without-zmq: + Werror: true + cmake_flags: + - '-DBUILD_BITCOIN_ZMQ=OFF' + targets: + - - all + - install + - - check + - check-functional + timeout: 1800 + + check-seeds: + script: builds/check-seeds.sh + templates: + - common_unix_artifacts + timeout: 600 + + check-source-control-tools: + cmake_flags: + - '-DBUILD_SOURCE_CONTROL_TOOLS=ON' + targets: + - - check-source-control-tools + timeout: 600 + + gitian-linux: + templates: + - gitian_builds + env: + OS_NAME: linux + + gitian-osx: + templates: + - gitian_builds + env: + OS_NAME: osx + + gitian-win: + templates: + - gitian_builds + env: + OS_NAME: win diff --git a/contrib/teamcity/setup-debian-buster.sh b/contrib/teamcity/setup-debian-buster.sh --- a/contrib/teamcity/setup-debian-buster.sh +++ b/contrib/teamcity/setup-debian-buster.sh @@ -59,6 +59,7 @@ python3-autopep8 python3-pip python3-setuptools + python3-yaml python3-zmq qemu-user-static qttools5-dev