Changeset View
Changeset View
Standalone View
Standalone View
contrib/teamcity/build-configurations.py
#!/usr/bin/env python3 | #!/usr/bin/env python3 | ||||
# Copyright (c) 2020 The Bitcoin developers | # Copyright (c) 2020 The Bitcoin developers | ||||
# Distributed under the MIT software license, see the accompanying | # Distributed under the MIT software license, see the accompanying | ||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php. | # file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||||
import argparse | import argparse | ||||
from deepmerge import always_merger | from deepmerge import always_merger | ||||
import json | import json | ||||
import os | import os | ||||
from pathlib import Path, PurePath | from pathlib import Path, PurePath | ||||
import signal | import signal | ||||
import shutil | |||||
import subprocess | import subprocess | ||||
import sys | import sys | ||||
from teamcity import is_running_under_teamcity | |||||
from teamcity.messages import TeamcityServiceMessages | from teamcity.messages import TeamcityServiceMessages | ||||
# Default timeout value in seconds. Should be overridden by the | # Default timeout value in seconds. Should be overridden by the | ||||
# configuration file. | # configuration file. | ||||
DEFAULT_TIMEOUT = 1 * 60 * 60 | DEFAULT_TIMEOUT = 1 * 60 * 60 | ||||
if sys.version_info < (3, 6): | if sys.version_info < (3, 6): | ||||
raise SystemError("This script requires python >= 3.6") | raise SystemError("This script requires python >= 3.6") | ||||
def copy_artifacts(teamcity_messages, build_dir, artifacts): | |||||
# This accounts for the volume mapping from the container. | |||||
# Our local /result is mapped to some relative ./results on the host, so we | |||||
jasonbcox: Nit: `/result` -> `/results` | |||||
# use /results/artifacts to copy our files but results/artifacts as an | |||||
# artifact path for teamcity. | |||||
# TODO abstract out the volume mapping | |||||
if is_running_under_teamcity(): | |||||
artifact_dir = Path("/results/artifacts") | |||||
else: | |||||
artifact_dir = build_dir.joinpath("artifacts") | |||||
if artifact_dir.is_dir(): | |||||
shutil.rmtree(artifact_dir) | |||||
artifact_dir.mkdir(exist_ok=True) | |||||
# Find and copy artifacts. | |||||
# The source is relative to the build tree, the destination relative to the | |||||
# artifact directory. | |||||
# The artifact directory is located in the build directory tree, results | |||||
# from it needs to be excluded from the glob matches to prevent infinite | |||||
# recursion. | |||||
for pattern, dest in artifacts.items(): | |||||
matches = [m for m in sorted(build_dir.glob( | |||||
pattern)) if artifact_dir not in m.parents and artifact_dir != m] | |||||
dest = artifact_dir.joinpath(dest) | |||||
# Pattern did not match | |||||
if not matches: | |||||
continue | |||||
jasonbcoxUnsubmitted Not Done Inline ActionsIt would be helpful for future investigations for this to print a warning and the entire pattern searched. jasonbcox: It would be helpful for future investigations for this to print a warning and the entire… | |||||
FabienAuthorUnsubmitted Done Inline ActionsThis is actually something you don't want in your log, glob is easy enough as a pattern that it's not worth it. You are kinda getting this today in the build log in the form of: cp: cannot stat '/tmp/sanitizer_logs': No such file or directory [22:37:43]W: [Step 1/1] cp: cannot stat '/work/build-without-wallet': No such file or directory [22:37:43]W: [Step 1/1] cp: cannot stat '/work/abc-ci-builds/build-without-wallet': No such file or directory [22:37:43]W: [Step 1/1] cp: cannot stat '/work/ibd/debug.log': No such file or directory at the end of the log, which is very confusing because it tells you that the cp command is the cause of failure, which it is not. Fabien: This is actually something you don't want in your log, glob is easy enough as a pattern that… | |||||
# If there is a single file, destination is the new file path | |||||
if len(matches) == 1 and matches[0].is_file(): | |||||
# Create the parent directories as needed | |||||
dest.parent.mkdir(parents=True, exist_ok=True) | |||||
shutil.copy2(matches[0], dest) | |||||
continue | |||||
# If there are multiple files or a single directory, destination is a | |||||
# directory. | |||||
dest.mkdir(parents=True, exist_ok=True) | |||||
for match in matches: | |||||
if match.is_file(): | |||||
shutil.copy2(match, dest) | |||||
else: | |||||
shutil.copytree(match, dest.joinpath(match.name)) | |||||
# Instruct teamcity to upload our artifact directory | |||||
artifact_path_pattern = "+:{}=>artifacts.tar.gz".format( | |||||
str(artifact_dir.relative_to("/")) | |||||
) | |||||
teamcity_messages.publishArtifacts(artifact_path_pattern) | |||||
def main(): | def main(): | ||||
script_dir = PurePath(os.path.realpath(__file__)).parent | script_dir = PurePath(os.path.realpath(__file__)).parent | ||||
# By default search for a configuration file in the same directory as this | # By default search for a configuration file in the same directory as this | ||||
# script. | # script. | ||||
default_config_path = Path( | default_config_path = Path( | ||||
script_dir.joinpath("build-configurations.json") | script_dir.joinpath("build-configurations.json") | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 107 Lines • ▼ Show 20 Lines | def main(): | ||||
# Let the user know what build is being run. | # Let the user know what build is being run. | ||||
# This makes it easier to retrieve the info from the logs. | # This makes it easier to retrieve the info from the logs. | ||||
teamcity_messages = TeamcityServiceMessages() | teamcity_messages = TeamcityServiceMessages() | ||||
teamcity_messages.customMessage( | teamcity_messages.customMessage( | ||||
"Starting build {}".format(args.build), | "Starting build {}".format(args.build), | ||||
status="NORMAL" | status="NORMAL" | ||||
) | ) | ||||
# Flag to indicate that the process and its children should be killed | |||||
kill_em_all = False | |||||
try: | try: | ||||
subprocess.run( | subprocess.run( | ||||
[str(script_path)] + unknown_args, | [str(script_path)] + unknown_args, | ||||
check=True, | check=True, | ||||
cwd=build_directory, | cwd=build_directory, | ||||
env={ | env={ | ||||
**os.environ, | **os.environ, | ||||
**environment_variables, | **environment_variables, | ||||
**build.get("environment", {}) | **build.get("environment", {}) | ||||
}, | }, | ||||
timeout=build.get("timeout", DEFAULT_TIMEOUT), | timeout=build.get("timeout", DEFAULT_TIMEOUT), | ||||
) | ) | ||||
except subprocess.TimeoutExpired as e: | except subprocess.TimeoutExpired as e: | ||||
print( | print( | ||||
"Build {} timed out after {:.1f}s".format( | "Build {} timed out after {:.1f}s".format( | ||||
args.build, round(e.timeout, 1) | args.build, round(e.timeout, 1) | ||||
) | ) | ||||
) | ) | ||||
# Make sure to kill all the child processes, as subprocess only kills | # Make sure to kill all the child processes, as subprocess only kills | ||||
# the one we started. It will also kill this python script ! | # the one we started. It will also kill this python script ! | ||||
# The return code is 128 + 9 (SIGKILL) = 137. | # The return code is 128 + 9 (SIGKILL) = 137. | ||||
os.killpg(os.getpgid(os.getpid()), signal.SIGKILL) | kill_em_all = True | ||||
except subprocess.CalledProcessError as e: | except subprocess.CalledProcessError as e: | ||||
print( | print( | ||||
"Build {} failed with exit code {}".format( | "Build {} failed with exit code {}".format( | ||||
args.build, | args.build, | ||||
e.returncode)) | e.returncode)) | ||||
sys.exit(e.returncode) | sys.exit(e.returncode) | ||||
finally: | |||||
copy_artifacts( | |||||
teamcity_messages, | |||||
build_directory, | |||||
build.get("artifacts", {}) | |||||
) | |||||
# Seek and destroy | |||||
if kill_em_all: | |||||
os.killpg(os.getpgid(os.getpid()), signal.SIGKILL) | |||||
if __name__ == '__main__': | if __name__ == '__main__': | ||||
main() | main() |
Nit: /result -> /results