Differential D6748 Diff 21810 contrib/teamcity/build-configurations.py

Changeset View

Standalone View

contrib/teamcity/build-configurations.py

#!/usr/bin/env python3		#!/usr/bin/env python3
# Copyright (c) 2020 The Bitcoin developers		# Copyright (c) 2020 The Bitcoin developers
# Distributed under the MIT software license, see the accompanying		# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.		# file COPYING or http://www.opensource.org/licenses/mit-license.php.

import argparse		import argparse
		import asyncio
from deepmerge import always_merger		from deepmerge import always_merger
import json		import json
import os		import os
from pathlib import Path, PurePath		from pathlib import Path, PurePath
import signal
import shutil		import shutil
import subprocess		import subprocess
import sys		import sys
from teamcity import is_running_under_teamcity		from teamcity import is_running_under_teamcity
from teamcity.messages import TeamcityServiceMessages		from teamcity.messages import TeamcityServiceMessages

# Default timeout value in seconds. Should be overridden by the		# Default timeout value in seconds. Should be overridden by the
# configuration file.		# configuration file.
DEFAULT_TIMEOUT = 1 * 60 * 60		DEFAULT_TIMEOUT = 1 * 60 * 60

if sys.version_info < (3, 6):		if sys.version_info < (3, 6):
raise SystemError("This script requires python >= 3.6")		raise SystemError("This script requires python >= 3.6")


def copy_artifacts(teamcity_messages, build_dir, artifacts):		def copy_artifacts(teamcity_messages, build_dir, artifacts):
# This accounts for the volume mapping from the container.		# This accounts for the volume mapping from the container.
# Our local /result is mapped to some relative ./results on the host, so we		# Our local /results is mapped to some relative ./results on the host, so we
# use /results/artifacts to copy our files but results/artifacts as an		# use /results/artifacts to copy our files but results/artifacts as an
# artifact path for teamcity.		# artifact path for teamcity.
# TODO abstract out the volume mapping		# TODO abstract out the volume mapping
if is_running_under_teamcity():		if is_running_under_teamcity():
artifact_dir = Path("/results/artifacts")		artifact_dir = Path("/results/artifacts")
else:		else:
artifact_dir = build_dir.joinpath("artifacts")		artifact_dir = build_dir.joinpath("artifacts")

▲ Show 20 Lines • Show All 157 Lines • ▼ Show 20 Lines	def main():
# Let the user know what build is being run.		# Let the user know what build is being run.
# This makes it easier to retrieve the info from the logs.		# This makes it easier to retrieve the info from the logs.
teamcity_messages = TeamcityServiceMessages()		teamcity_messages = TeamcityServiceMessages()
teamcity_messages.customMessage(		teamcity_messages.customMessage(
"Starting build {}".format(args.build),		"Starting build {}".format(args.build),
status="NORMAL"		status="NORMAL"
)		)

# Flag to indicate that the process and its children should be killed		# Build 2 log files:
kill_em_all = False		# - the full log will contain all unfiltered content
		# - the clean log will contain the same filtered content as what is printed
try:		# to stdout. This filter is done in print_line_to_logs().
subprocess.run(		clean_log = build_directory.joinpath("build.clean.log")
[str(script_path)] + unknown_args,		if clean_log.is_file():
check=True,		clean_log.unlink()

		full_log = build_directory.joinpath("build.full.log")
		if full_log.is_file():
		full_log.unlink()

		def print_line_to_logs(line):
		# Always print to the full log
		with open(full_log, 'a', encoding='utf-8') as log:
		log.write(line)

		# Discard the set -x bash output for stdout and the clean log
		if not line.startswith("+"):
		with open(clean_log, 'a', encoding='utf-8') as log:
		log.write(line)
		print(line.rstrip())

		async def process_stdout(stdout):
		while True:
		line = await stdout.readline()
		line = line.decode('utf-8')

		if not line:
		break

		print_line_to_logs(line)

		async def run_build():
		proc = await asyncio.create_subprocess_exec(
		*([str(script_path)] + unknown_args),
		stdout=asyncio.subprocess.PIPE,
		stderr=asyncio.subprocess.STDOUT,
cwd=build_directory,		cwd=build_directory,
env={		env={
**os.environ,		**os.environ,
**environment_variables,		**environment_variables,
**build.get("environment", {})		**build.get("environment", {})
},		},
timeout=build.get("timeout", DEFAULT_TIMEOUT),
)		)
except subprocess.TimeoutExpired as e:
print(		await asyncio.wait([
		process_stdout(proc.stdout)
		])

		return await proc.wait()

		async def wait_for_build(timeout):
		try:
		return_code = await asyncio.wait_for(run_build(), timeout)
		if return_code != 0:
		print_line_to_logs(
		"Build {} failed with exit code {}".format(
		args.build,
		return_code
		)
		)
		except asyncio.TimeoutError:
		print_line_to_logs(
"Build {} timed out after {:.1f}s".format(		"Build {} timed out after {:.1f}s".format(
args.build, round(e.timeout, 1)		args.build, round(timeout, 1)
)		)
)		)
# Make sure to kill all the child processes, as subprocess only kills		# The process is killed, set return code to 128 + 9 (SIGKILL) = 137
# the one we started. It will also kill this python script !		return_code = 137
# The return code is 128 + 9 (SIGKILL) = 137.
kill_em_all = True
except subprocess.CalledProcessError as e:
print(
"Build {} failed with exit code {}".format(
args.build,
e.returncode))
sys.exit(e.returncode)
finally:		finally:
		# Always add the build logs to the root of the artifacts
		artifacts = {
		**build.get("artifacts", {}),
		str(full_log.relative_to(build_directory)): "",
		str(clean_log.relative_to(build_directory)): "",
		}

copy_artifacts(		copy_artifacts(
teamcity_messages,		teamcity_messages,
build_directory,		build_directory,
build.get("artifacts", {})		artifacts
		)

		return return_code

		return_code = asyncio.run(
		wait_for_build(build.get("timeout", DEFAULT_TIMEOUT))
)		)

# Seek and destroy		sys.exit(return_code)
if kill_em_all:
os.killpg(os.getpgid(os.getpid()), signal.SIGKILL)


if __name__ == '__main__':		if __name__ == '__main__':
main()		main()