Differential D6735 Diff 21809 contrib/teamcity/build-configurations.py

Changeset View

Standalone View

contrib/teamcity/build-configurations.py

#!/usr/bin/env python3		#!/usr/bin/env python3
# Copyright (c) 2020 The Bitcoin developers		# Copyright (c) 2020 The Bitcoin developers
# Distributed under the MIT software license, see the accompanying		# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.		# file COPYING or http://www.opensource.org/licenses/mit-license.php.

import argparse		import argparse
from deepmerge import always_merger		from deepmerge import always_merger
import json		import json
import os		import os
from pathlib import Path, PurePath		from pathlib import Path, PurePath
import signal		import signal
		import shutil
import subprocess		import subprocess
import sys		import sys
		from teamcity import is_running_under_teamcity
from teamcity.messages import TeamcityServiceMessages		from teamcity.messages import TeamcityServiceMessages

# Default timeout value in seconds. Should be overridden by the		# Default timeout value in seconds. Should be overridden by the
# configuration file.		# configuration file.
DEFAULT_TIMEOUT = 1 * 60 * 60		DEFAULT_TIMEOUT = 1 * 60 * 60

if sys.version_info < (3, 6):		if sys.version_info < (3, 6):
raise SystemError("This script requires python >= 3.6")		raise SystemError("This script requires python >= 3.6")


		def copy_artifacts(teamcity_messages, build_dir, artifacts):
		# This accounts for the volume mapping from the container.
		# Our local /result is mapped to some relative ./results on the host, so we
		# use /results/artifacts to copy our files but results/artifacts as an
		# artifact path for teamcity.
		# TODO abstract out the volume mapping
		if is_running_under_teamcity():
		artifact_dir = Path("/results/artifacts")
		else:
		artifact_dir = build_dir.joinpath("artifacts")

		if artifact_dir.is_dir():
		shutil.rmtree(artifact_dir)
		artifact_dir.mkdir(exist_ok=True)

		# Find and copy artifacts.
		# The source is relative to the build tree, the destination relative to the
		# artifact directory.
		# The artifact directory is located in the build directory tree, results
		# from it needs to be excluded from the glob matches to prevent infinite
		# recursion.
		for pattern, dest in artifacts.items():
		matches = [m for m in sorted(build_dir.glob(
		pattern)) if artifact_dir not in m.parents and artifact_dir != m]
		dest = artifact_dir.joinpath(dest)

		# Pattern did not match
		if not matches:
		continue

		# If there is a single file, destination is the new file path
		if len(matches) == 1 and matches[0].is_file():
		# Create the parent directories as needed
		dest.parent.mkdir(parents=True, exist_ok=True)
		shutil.copy2(matches[0], dest)
		continue

		# If there are multiple files or a single directory, destination is a
		# directory.
		dest.mkdir(parents=True, exist_ok=True)
		for match in matches:
		if match.is_file():
		shutil.copy2(match, dest)
		else:
		shutil.copytree(match, dest.joinpath(match.name))

		# Instruct teamcity to upload our artifact directory
		artifact_path_pattern = "+:{}=>artifacts.tar.gz".format(
		str(artifact_dir.relative_to("/"))
		)
		teamcity_messages.publishArtifacts(artifact_path_pattern)


def main():		def main():
script_dir = PurePath(os.path.realpath(__file__)).parent		script_dir = PurePath(os.path.realpath(__file__)).parent

# By default search for a configuration file in the same directory as this		# By default search for a configuration file in the same directory as this
# script.		# script.
default_config_path = Path(		default_config_path = Path(
script_dir.joinpath("build-configurations.json")		script_dir.joinpath("build-configurations.json")
)		)
▲ Show 20 Lines • Show All 107 Lines • ▼ Show 20 Lines	def main():
# Let the user know what build is being run.		# Let the user know what build is being run.
# This makes it easier to retrieve the info from the logs.		# This makes it easier to retrieve the info from the logs.
teamcity_messages = TeamcityServiceMessages()		teamcity_messages = TeamcityServiceMessages()
teamcity_messages.customMessage(		teamcity_messages.customMessage(
"Starting build {}".format(args.build),		"Starting build {}".format(args.build),
status="NORMAL"		status="NORMAL"
)		)

		# Flag to indicate that the process and its children should be killed
		kill_em_all = False

try:		try:
subprocess.run(		subprocess.run(
[str(script_path)] + unknown_args,		[str(script_path)] + unknown_args,
check=True,		check=True,
cwd=build_directory,		cwd=build_directory,
env={		env={
**os.environ,		**os.environ,
**environment_variables,		**environment_variables,
**build.get("environment", {})		**build.get("environment", {})
},		},
timeout=build.get("timeout", DEFAULT_TIMEOUT),		timeout=build.get("timeout", DEFAULT_TIMEOUT),
)		)
except subprocess.TimeoutExpired as e:		except subprocess.TimeoutExpired as e:
print(		print(
"Build {} timed out after {:.1f}s".format(		"Build {} timed out after {:.1f}s".format(
args.build, round(e.timeout, 1)		args.build, round(e.timeout, 1)
)		)
)		)
# Make sure to kill all the child processes, as subprocess only kills		# Make sure to kill all the child processes, as subprocess only kills
# the one we started. It will also kill this python script !		# the one we started. It will also kill this python script !
# The return code is 128 + 9 (SIGKILL) = 137.		# The return code is 128 + 9 (SIGKILL) = 137.
os.killpg(os.getpgid(os.getpid()), signal.SIGKILL)		kill_em_all = True
except subprocess.CalledProcessError as e:		except subprocess.CalledProcessError as e:
print(		print(
"Build {} failed with exit code {}".format(		"Build {} failed with exit code {}".format(
args.build,		args.build,
e.returncode))		e.returncode))
sys.exit(e.returncode)		sys.exit(e.returncode)
		finally:
		copy_artifacts(
		teamcity_messages,
		build_directory,
		build.get("artifacts", {})
		)

		# Seek and destroy
		if kill_em_all:
		os.killpg(os.getpgid(os.getpid()), signal.SIGKILL)


if __name__ == '__main__':		if __name__ == '__main__':
main()		main()