Differential D6750 Diff 21800 contrib/teamcity/build-configurations.py

Changeset View

Standalone View

contrib/teamcity/build-configurations.py

Show First 20 Lines • Show All 42 Lines • ▼ Show 20 Lines	class BuildConfiguration:

def load(self, build_name):		def load(self, build_name):
self.name = build_name		self.name = build_name

# Read the configuration		# Read the configuration
with open(self.config_file, encoding="utf-8") as f:		with open(self.config_file, encoding="utf-8") as f:
config = json.load(f)		config = json.load(f)

# The configuration root should contain a mandatory element "builds", and it		# The configuration root should contain a mandatory element "builds", and
# should not be empty.		# it should not be empty.
if not config.get("builds", None):		if not config.get("builds", None):
raise AssertionError(		raise AssertionError(
"Invalid configuration file {}: the \"builds\" element is missing or empty".format(		"Invalid configuration file {}: the \"builds\" element is missing or empty".format(
str(self.config_file)		str(self.config_file)
)		)
)		)

# Check the target build has an entry in the configuration file		# Check the target build has an entry in the configuration file
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines	def load(self, build_name):
str(self.script_path)		str(self.script_path)
)		)
)		)

def get(self, key, default):		def get(self, key, default):
return self.config.get(key, default)		return self.config.get(key, default)


def copy_artifacts(teamcity_messages, build_dir, artifacts):		class UserBuild():
# This accounts for the volume mapping from the container.		def __init__(self, configuration, project_root):
# Our local /results is mapped to some relative ./results on the host, so we		self.configuration = configuration
# use /results/artifacts to copy our files but results/artifacts as an		self.project_root = project_root
# artifact path for teamcity.
# TODO abstract out the volume mapping
if is_running_under_teamcity():
artifact_dir = Path("/results/artifacts")
else:
artifact_dir = build_dir.joinpath("artifacts")

if artifact_dir.is_dir():		# Create the build directory as needed
shutil.rmtree(artifact_dir)		self.build_directory = Path(
artifact_dir.mkdir(exist_ok=True)		self.project_root.joinpath(
		'abc-ci-builds',
		self.configuration.name))
		self.build_directory.mkdir(exist_ok=True, parents=True)

		self.artifact_dir = self.build_directory.joinpath("artifacts")

		# We will provide the required environment variables
		self.environment_variables = {
		"BUILD_DIR": str(self.build_directory),
		"CMAKE_PLATFORMS_DIR": self.project_root.joinpath("cmake", "platforms"),
		"THREADS": str(os.cpu_count() or 1),
		"TOPLEVEL": str(self.project_root),
		}

		# Build 2 log files:
		# - the full log will contain all unfiltered content
		# - the clean log will contain the same filtered content as what is
		# printed to stdout. This filter is done in print_line_to_logs().
		self.logs = {}
		self.logs["clean_log"] = self.build_directory.joinpath(
		"build.clean.log")
		if self.logs["clean_log"].is_file():
		self.logs["clean_log"].unlink()

		self.logs["full_log"] = self.build_directory.joinpath("build.full.log")
		if self.logs["full_log"].is_file():
		self.logs["full_log"].unlink()

		def copy_artifacts(self, artifacts):
		if self.artifact_dir.is_dir():
		shutil.rmtree(self.artifact_dir)
		self.artifact_dir.mkdir(exist_ok=True)

# Find and copy artifacts.		# Find and copy artifacts.
# The source is relative to the build tree, the destination relative to the		# The source is relative to the build tree, the destination relative to
# artifact directory.		# the artifact directory.
# The artifact directory is located in the build directory tree, results		# The artifact directory is located in the build directory tree, results
# from it needs to be excluded from the glob matches to prevent infinite		# from it needs to be excluded from the glob matches to prevent infinite
# recursion.		# recursion.
for pattern, dest in artifacts.items():		for pattern, dest in artifacts.items():
matches = [m for m in sorted(build_dir.glob(		matches = [m for m in sorted(self.build_directory.glob(
pattern)) if artifact_dir not in m.parents and artifact_dir != m]		pattern)) if self.artifact_dir not in m.parents and self.artifact_dir != m]
dest = artifact_dir.joinpath(dest)		dest = self.artifact_dir.joinpath(dest)

# Pattern did not match		# Pattern did not match
if not matches:		if not matches:
continue		continue

# If there is a single file, destination is the new file path		# If there is a single file, destination is the new file path
if len(matches) == 1 and matches[0].is_file():		if len(matches) == 1 and matches[0].is_file():
# Create the parent directories as needed		# Create the parent directories as needed
dest.parent.mkdir(parents=True, exist_ok=True)		dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(matches[0], dest)		shutil.copy2(matches[0], dest)
continue		continue

# If there are multiple files or a single directory, destination is a		# If there are multiple files or a single directory, destination is a
# directory.		# directory.
dest.mkdir(parents=True, exist_ok=True)		dest.mkdir(parents=True, exist_ok=True)
for match in matches:		for match in matches:
if match.is_file():		if match.is_file():
shutil.copy2(match, dest)		shutil.copy2(match, dest)
else:		else:
shutil.copytree(match, dest.joinpath(match.name))		shutil.copytree(match, dest.joinpath(match.name))

# Instruct teamcity to upload our artifact directory		def print_line_to_logs(self, line):
artifact_path_pattern = "+:{}=>artifacts.tar.gz".format(
str(artifact_dir.relative_to("/"))
)
teamcity_messages.publishArtifacts(artifact_path_pattern)


def main():
script_dir = PurePath(os.path.realpath(__file__)).parent

# By default search for a configuration file in the same directory as this
# script.
default_config_path = Path(
script_dir.joinpath("build-configurations.json")
)

parser = argparse.ArgumentParser(description="Run a CI build")
parser.add_argument(
"build",
help="The name of the build to run"
)
parser.add_argument(
"--config",
"-c",
help="Path to the builds configuration file (default to {})".format(
str(default_config_path)
)
)

args, unknown_args = parser.parse_known_args()

# Check the configuration file exists
config_path = Path(args.config) if args.config else default_config_path
build_configuration = BuildConfiguration(
script_dir, config_path, args.build)

# Find the git root directory
git_root = PurePath(
subprocess.run(
['git', 'rev-parse', '--show-toplevel'],
capture_output=True,
check=True,
encoding='utf-8',
text=True,
).stdout.strip()
)

# Create the build directory as needed
build_directory = Path(
git_root.joinpath(
'abc-ci-builds',
build_configuration.name))
build_directory.mkdir(exist_ok=True, parents=True)

# We will provide the required environment variables
environment_variables = {
"BUILD_DIR": str(build_directory),
"CMAKE_PLATFORMS_DIR": git_root.joinpath("cmake", "platforms"),
"THREADS": str(os.cpu_count() or 1),
"TOPLEVEL": str(git_root),
}

# Let the user know what build is being run.
# This makes it easier to retrieve the info from the logs.
teamcity_messages = TeamcityServiceMessages()
teamcity_messages.customMessage(
"Starting build {}".format(build_configuration.name),
status="NORMAL"
)

# Build 2 log files:
# - the full log will contain all unfiltered content
# - the clean log will contain the same filtered content as what is printed
# to stdout. This filter is done in print_line_to_logs().
clean_log = build_directory.joinpath("build.clean.log")
if clean_log.is_file():
clean_log.unlink()

full_log = build_directory.joinpath("build.full.log")
if full_log.is_file():
full_log.unlink()

def print_line_to_logs(line):
# Always print to the full log		# Always print to the full log
with open(full_log, 'a', encoding='utf-8') as log:		with open(self.logs["full_log"], 'a', encoding='utf-8') as log:
log.write(line)		log.write(line)

# Discard the set -x bash output for stdout and the clean log		# Discard the set -x bash output for stdout and the clean log
if not line.startswith("+"):		if not line.startswith("+"):
with open(clean_log, 'a', encoding='utf-8') as log:		with open(self.logs["clean_log"], 'a', encoding='utf-8') as log:
log.write(line)		log.write(line)
print(line.rstrip())		print(line.rstrip())

async def process_stdout(stdout):		async def process_stdout(self, stdout):
while True:		while True:
line = await stdout.readline()		line = await stdout.readline()
line = line.decode('utf-8')		line = line.decode('utf-8')

if not line:		if not line:
break		break

print_line_to_logs(line)		self.print_line_to_logs(line)

async def run_build():		async def run_build(self, args=[]):
proc = await asyncio.create_subprocess_exec(		proc = await asyncio.create_subprocess_exec(
*([str(build_configuration.script_path)] + unknown_args),		*([str(self.configuration.script_path)] + args),
stdout=asyncio.subprocess.PIPE,		stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,		stderr=asyncio.subprocess.STDOUT,
cwd=build_directory,		cwd=self.build_directory,
env={		env={
**os.environ,		**os.environ,
**environment_variables,		**self.environment_variables,
**build_configuration.get("environment", {})		**self.configuration.get("environment", {})
},		},
)		)

await asyncio.wait([		await asyncio.wait([
process_stdout(proc.stdout)		self.process_stdout(proc.stdout)
])		])

return await proc.wait()		return await proc.wait()

async def wait_for_build(timeout):		async def wait_for_build(self, timeout, args=[]):
try:		try:
return_code = await asyncio.wait_for(run_build(), timeout)		return_code = await asyncio.wait_for(self.run_build(args), timeout)
if return_code != 0:		if return_code != 0:
print_line_to_logs(		self.print_line_to_logs(
"Build {} failed with exit code {}".format(		"Build {} failed with exit code {}".format(
build_configuration.name,		self.configuration.name,
return_code		return_code
)		)
)		)
except asyncio.TimeoutError:		except asyncio.TimeoutError:
print_line_to_logs(		self.print_line_to_logs(
"Build {} timed out after {:.1f}s".format(		"Build {} timed out after {:.1f}s".format(
build_configuration.name, round(timeout, 1)		self.configuration.name, round(timeout, 1)
)		)
)		)
# The process is killed, set return code to 128 + 9 (SIGKILL) = 137		# The process is killed, set return code to 128 + 9 (SIGKILL) = 137
return_code = 137		return_code = 137
finally:		finally:
# Always add the build logs to the root of the artifacts		# Always add the build logs to the root of the artifacts
artifacts = {		artifacts = {
**build_configuration.get("artifacts", {}),		**self.configuration.get("artifacts", {}),
str(full_log.relative_to(build_directory)): "",		str(self.logs["full_log"].relative_to(self.build_directory)): "",
str(clean_log.relative_to(build_directory)): "",		str(self.logs["clean_log"].relative_to(self.build_directory)): "",
}		}

copy_artifacts(		self.copy_artifacts(artifacts)
teamcity_messages,
build_directory,
artifacts
)

return return_code		return return_code

return_code = asyncio.run(		def run(self, args=[]):
wait_for_build(build_configuration.get("timeout", DEFAULT_TIMEOUT))		return asyncio.run(
		self.wait_for_build(
		self.configuration.get(
		"timeout", DEFAULT_TIMEOUT))
		)


		class TeamcityBuild(UserBuild):
		def __init__(self, configuration, project_root):
		super().__init__(configuration, project_root)

		# This accounts for the volume mapping from the container.
		# Our local /results is mapped to some relative ./results on the host,
		# so we use /results/artifacts to copy our files but results/artifacts as
		# an artifact path for teamcity.
		# TODO abstract out the volume mapping
		self.artifact_dir = Path("/results/artifacts")

		self.teamcity_messages = TeamcityServiceMessages()

		def copy_artifacts(self, artifacts):
		super().copy_artifacts(artifacts)

		# Instruct teamcity to upload our artifact directory
		artifact_path_pattern = "+:{}=>artifacts.tar.gz".format(
		str(self.artifact_dir.relative_to("/"))
		)
		self.teamcity_messages.publishArtifacts(artifact_path_pattern)

		def run(self, args=[]):
		# Let the user know what build is being run.
		# This makes it easier to retrieve the info from the logs.
		self.teamcity_messages.customMessage(
		"Starting build {}".format(self.configuration.name),
		status="NORMAL"
		)

		return super().run()


		def main():
		script_dir = PurePath(os.path.realpath(__file__)).parent

		# By default search for a configuration file in the same directory as this
		# script.
		default_config_path = Path(
		script_dir.joinpath("build-configurations.json")
)		)

sys.exit(return_code)		parser = argparse.ArgumentParser(description="Run a CI build")
		parser.add_argument(
		"build",
		help="The name of the build to run"
		)
		parser.add_argument(
		"--config",
		"-c",
		help="Path to the builds configuration file (default to {})".format(
		str(default_config_path)
		)
		)

		args, unknown_args = parser.parse_known_args()

		# Check the configuration file exists
		config_path = Path(args.config) if args.config else default_config_path
		build_configuration = BuildConfiguration(
		script_dir, config_path, args.build)

		git_root = PurePath(
		subprocess.run(
		['git', 'rev-parse', '--show-toplevel'],
		capture_output=True,
		check=True,
		encoding='utf-8',
		text=True,
		).stdout.strip()
		)

		if is_running_under_teamcity():
		build = TeamcityBuild(build_configuration, git_root)
		else:
		build = UserBuild(build_configuration, git_root)

		sys.exit(build.run(unknown_args))


if __name__ == '__main__':		if __name__ == '__main__':
main()		main()