Changeset View
Changeset View
Standalone View
Standalone View
contrib/teamcity/build-configurations.py
Show All 17 Lines | |||||
# Default timeout value in seconds. Should be overridden by the | # Default timeout value in seconds. Should be overridden by the | ||||
# configuration file. | # configuration file. | ||||
DEFAULT_TIMEOUT = 1 * 60 * 60 | DEFAULT_TIMEOUT = 1 * 60 * 60 | ||||
if sys.version_info < (3, 6): | if sys.version_info < (3, 6): | ||||
raise SystemError("This script requires python >= 3.6") | raise SystemError("This script requires python >= 3.6") | ||||
class BuildConfiguration: | |||||
def __init__(self, script_root, config_file, build_name=None): | |||||
self.script_root = script_root | |||||
self.config_file = config_file | |||||
self.name = None | |||||
self.config = {} | |||||
self.script_path = Path() | |||||
if not config_file.is_file(): | |||||
raise FileNotFoundError( | |||||
"The configuration file does not exist {}".format( | |||||
str(config_file) | |||||
) | |||||
) | |||||
if build_name is not None: | |||||
self.load(build_name) | |||||
def load(self, build_name): | |||||
self.name = build_name | |||||
# Read the configuration | |||||
with open(self.config_file, encoding="utf-8") as f: | |||||
config = json.load(f) | |||||
# The configuration root should contain a mandatory element "builds", and it | |||||
# should not be empty. | |||||
if not config.get("builds", None): | |||||
raise AssertionError( | |||||
"Invalid configuration file {}: the \"builds\" element is missing or empty".format( | |||||
str(self.config_file) | |||||
) | |||||
) | |||||
# Check the target build has an entry in the configuration file | |||||
build = config["builds"].get(self.name, None) | |||||
if not build: | |||||
raise AssertionError( | |||||
"{} is not a valid build identifier. Valid identifiers are {}".format( | |||||
self.name, list(config.keys()) | |||||
) | |||||
) | |||||
# Get a list of the templates, if any | |||||
templates = config.get("templates", {}) | |||||
# If the build references a template, merge the configurations | |||||
template_name = build.get("template", None) | |||||
if template_name: | |||||
# Raise an error if the template does not exist | |||||
if template_name not in templates: | |||||
raise AssertionError( | |||||
"Build {} configuration inherits from template {}, but the template does not exist.".format( | |||||
self.name, | |||||
template_name | |||||
) | |||||
) | |||||
# The template exists, apply the build configuration on top of the | |||||
# template | |||||
self.config = always_merger.merge( | |||||
templates.get(template_name, {}), build) | |||||
# Make sure there is a script file associated with the build... | |||||
script = build.get("script", None) | |||||
if script is None: | |||||
raise AssertionError( | |||||
"No script provided for the build {}".format( | |||||
self.name | |||||
) | |||||
) | |||||
# ... and that the script file can be executed | |||||
self.script_path = Path(self.script_root.joinpath(script)) | |||||
if not self.script_path.is_file() or not os.access(self.script_path, os.X_OK): | |||||
raise FileNotFoundError( | |||||
"The script file {} does not exist or does not have execution permission".format( | |||||
str(self.script_path) | |||||
) | |||||
) | |||||
def get(self, key, default): | |||||
return self.config.get(key, default) | |||||
def copy_artifacts(teamcity_messages, build_dir, artifacts): | def copy_artifacts(teamcity_messages, build_dir, artifacts): | ||||
# This accounts for the volume mapping from the container. | # This accounts for the volume mapping from the container. | ||||
# Our local /results is mapped to some relative ./results on the host, so we | # Our local /results is mapped to some relative ./results on the host, so we | ||||
# use /results/artifacts to copy our files but results/artifacts as an | # use /results/artifacts to copy our files but results/artifacts as an | ||||
# artifact path for teamcity. | # artifact path for teamcity. | ||||
# TODO abstract out the volume mapping | # TODO abstract out the volume mapping | ||||
if is_running_under_teamcity(): | if is_running_under_teamcity(): | ||||
artifact_dir = Path("/results/artifacts") | artifact_dir = Path("/results/artifacts") | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | parser.add_argument( | ||||
str(default_config_path) | str(default_config_path) | ||||
) | ) | ||||
) | ) | ||||
args, unknown_args = parser.parse_known_args() | args, unknown_args = parser.parse_known_args() | ||||
# Check the configuration file exists | # Check the configuration file exists | ||||
config_path = Path(args.config) if args.config else default_config_path | config_path = Path(args.config) if args.config else default_config_path | ||||
if not config_path.is_file(): | build_configuration = BuildConfiguration( | ||||
raise FileNotFoundError( | script_dir, config_path, args.build) | ||||
"The configuration file does not exist {}".format( | |||||
str(config_path) | |||||
) | |||||
) | |||||
# Read the configuration | |||||
with open(config_path, encoding="utf-8") as f: | |||||
config = json.load(f) | |||||
# The configuration root should contain a mandatory element "builds", and it | |||||
# should not be empty. | |||||
if not config.get("builds", None): | |||||
raise AssertionError( | |||||
"Invalid configuration file {}: the \"builds\" element is missing or empty".format( | |||||
str(config_path) | |||||
) | |||||
) | |||||
# Check the target build has an entry in the configuration file | |||||
build = config["builds"].get(args.build, None) | |||||
if not build: | |||||
raise AssertionError( | |||||
"{} is not a valid build identifier. Valid identifiers are {}".format( | |||||
args.build, list(config.keys()) | |||||
) | |||||
) | |||||
# Get a list of the templates, if any | |||||
templates = config.get("templates", {}) | |||||
# If the build references a template, merge the configurations | |||||
template_name = build.get("template", None) | |||||
if template_name: | |||||
# Raise an error if the template does not exist | |||||
if template_name not in templates: | |||||
raise AssertionError( | |||||
"Build {} configuration inherits from template {}, but the template does not exist.".format( | |||||
args.build, | |||||
template_name | |||||
) | |||||
) | |||||
# The template exists, apply the build configuration on top of the | |||||
# template | |||||
build = always_merger.merge(templates.get(template_name, {}), build) | |||||
# Make sure there is a script file associated with the build... | |||||
script = build.get("script", None) | |||||
if script is None: | |||||
raise AssertionError( | |||||
"No script provided for the build {}".format( | |||||
args.build | |||||
) | |||||
) | |||||
# ... and that the script file can be executed | |||||
script_path = Path(script_dir.joinpath(script)) | |||||
if not script_path.is_file() or not os.access(script_path, os.X_OK): | |||||
raise FileNotFoundError( | |||||
"The script file {} does not exist or does not have execution permission".format( | |||||
str(script_path) | |||||
) | |||||
) | |||||
# Find the git root directory | # Find the git root directory | ||||
git_root = PurePath( | git_root = PurePath( | ||||
subprocess.run( | subprocess.run( | ||||
['git', 'rev-parse', '--show-toplevel'], | ['git', 'rev-parse', '--show-toplevel'], | ||||
capture_output=True, | capture_output=True, | ||||
check=True, | check=True, | ||||
encoding='utf-8', | encoding='utf-8', | ||||
text=True, | text=True, | ||||
).stdout.strip() | ).stdout.strip() | ||||
) | ) | ||||
# Create the build directory as needed | # Create the build directory as needed | ||||
build_directory = Path(git_root.joinpath('abc-ci-builds', args.build)) | build_directory = Path( | ||||
git_root.joinpath( | |||||
'abc-ci-builds', | |||||
build_configuration.name)) | |||||
build_directory.mkdir(exist_ok=True, parents=True) | build_directory.mkdir(exist_ok=True, parents=True) | ||||
# We will provide the required environment variables | # We will provide the required environment variables | ||||
environment_variables = { | environment_variables = { | ||||
"BUILD_DIR": str(build_directory), | "BUILD_DIR": str(build_directory), | ||||
"CMAKE_PLATFORMS_DIR": git_root.joinpath("cmake", "platforms"), | "CMAKE_PLATFORMS_DIR": git_root.joinpath("cmake", "platforms"), | ||||
"THREADS": str(os.cpu_count() or 1), | "THREADS": str(os.cpu_count() or 1), | ||||
"TOPLEVEL": str(git_root), | "TOPLEVEL": str(git_root), | ||||
} | } | ||||
# Let the user know what build is being run. | # Let the user know what build is being run. | ||||
# This makes it easier to retrieve the info from the logs. | # This makes it easier to retrieve the info from the logs. | ||||
teamcity_messages = TeamcityServiceMessages() | teamcity_messages = TeamcityServiceMessages() | ||||
teamcity_messages.customMessage( | teamcity_messages.customMessage( | ||||
"Starting build {}".format(args.build), | "Starting build {}".format(build_configuration.name), | ||||
status="NORMAL" | status="NORMAL" | ||||
) | ) | ||||
# Build 2 log files: | # Build 2 log files: | ||||
# - the full log will contain all unfiltered content | # - the full log will contain all unfiltered content | ||||
# - the clean log will contain the same filtered content as what is printed | # - the clean log will contain the same filtered content as what is printed | ||||
# to stdout. This filter is done in print_line_to_logs(). | # to stdout. This filter is done in print_line_to_logs(). | ||||
clean_log = build_directory.joinpath("build.clean.log") | clean_log = build_directory.joinpath("build.clean.log") | ||||
Show All 22 Lines | async def process_stdout(stdout): | ||||
if not line: | if not line: | ||||
break | break | ||||
print_line_to_logs(line) | print_line_to_logs(line) | ||||
async def run_build(): | async def run_build(): | ||||
proc = await asyncio.create_subprocess_exec( | proc = await asyncio.create_subprocess_exec( | ||||
*([str(script_path)] + unknown_args), | *([str(build_configuration.script_path)] + unknown_args), | ||||
stdout=asyncio.subprocess.PIPE, | stdout=asyncio.subprocess.PIPE, | ||||
stderr=asyncio.subprocess.STDOUT, | stderr=asyncio.subprocess.STDOUT, | ||||
cwd=build_directory, | cwd=build_directory, | ||||
env={ | env={ | ||||
**os.environ, | **os.environ, | ||||
**environment_variables, | **environment_variables, | ||||
**build.get("environment", {}) | **build_configuration.get("environment", {}) | ||||
}, | }, | ||||
) | ) | ||||
await asyncio.wait([ | await asyncio.wait([ | ||||
process_stdout(proc.stdout) | process_stdout(proc.stdout) | ||||
]) | ]) | ||||
return await proc.wait() | return await proc.wait() | ||||
async def wait_for_build(timeout): | async def wait_for_build(timeout): | ||||
try: | try: | ||||
return_code = await asyncio.wait_for(run_build(), timeout) | return_code = await asyncio.wait_for(run_build(), timeout) | ||||
if return_code != 0: | if return_code != 0: | ||||
print_line_to_logs( | print_line_to_logs( | ||||
"Build {} failed with exit code {}".format( | "Build {} failed with exit code {}".format( | ||||
args.build, | build_configuration.name, | ||||
return_code | return_code | ||||
) | ) | ||||
) | ) | ||||
except asyncio.TimeoutError: | except asyncio.TimeoutError: | ||||
print_line_to_logs( | print_line_to_logs( | ||||
"Build {} timed out after {:.1f}s".format( | "Build {} timed out after {:.1f}s".format( | ||||
args.build, round(timeout, 1) | build_configuration.name, round(timeout, 1) | ||||
) | ) | ||||
) | ) | ||||
# The process is killed, set return code to 128 + 9 (SIGKILL) = 137 | # The process is killed, set return code to 128 + 9 (SIGKILL) = 137 | ||||
return_code = 137 | return_code = 137 | ||||
finally: | finally: | ||||
# Always add the build logs to the root of the artifacts | # Always add the build logs to the root of the artifacts | ||||
artifacts = { | artifacts = { | ||||
**build.get("artifacts", {}), | **build_configuration.get("artifacts", {}), | ||||
str(full_log.relative_to(build_directory)): "", | str(full_log.relative_to(build_directory)): "", | ||||
str(clean_log.relative_to(build_directory)): "", | str(clean_log.relative_to(build_directory)): "", | ||||
} | } | ||||
copy_artifacts( | copy_artifacts( | ||||
teamcity_messages, | teamcity_messages, | ||||
build_directory, | build_directory, | ||||
artifacts | artifacts | ||||
) | ) | ||||
return return_code | return return_code | ||||
return_code = asyncio.run( | return_code = asyncio.run( | ||||
wait_for_build(build.get("timeout", DEFAULT_TIMEOUT)) | wait_for_build(build_configuration.get("timeout", DEFAULT_TIMEOUT)) | ||||
) | ) | ||||
sys.exit(return_code) | sys.exit(return_code) | ||||
if __name__ == '__main__': | if __name__ == '__main__': | ||||
main() | main() |