Changeset View
Changeset View
Standalone View
Standalone View
contrib/teamcity/build-configurations.py
Show All 23 Lines | |||||
class BuildConfiguration: | class BuildConfiguration: | ||||
def __init__(self, script_root, config_file, build_name=None): | def __init__(self, script_root, config_file, build_name=None): | ||||
self.script_root = script_root | self.script_root = script_root | ||||
self.config_file = config_file | self.config_file = config_file | ||||
self.name = None | self.name = None | ||||
self.config = {} | self.config = {} | ||||
self.script_path = Path() | self.cmake_flags = [] | ||||
self.build_steps = [] | |||||
self.build_directory = None | self.build_directory = None | ||||
self.junit_reports_dir = None | self.junit_reports_dir = None | ||||
self.test_logs_dir = None | self.test_logs_dir = None | ||||
self.project_root = PurePath( | self.project_root = PurePath( | ||||
subprocess.run( | subprocess.run( | ||||
['git', 'rev-parse', '--show-toplevel'], | ['git', 'rev-parse', '--show-toplevel'], | ||||
capture_output=True, | capture_output=True, | ||||
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | def load(self, build_name): | ||||
self.name, | self.name, | ||||
template_name | template_name | ||||
) | ) | ||||
) | ) | ||||
always_merger.merge(template_config, templates.get(template_name)) | always_merger.merge(template_config, templates.get(template_name)) | ||||
self.config = always_merger.merge(template_config, build) | self.config = always_merger.merge(template_config, build) | ||||
# Make sure there is a script file associated with the build... | |||||
script = self.config.get("script", None) | |||||
if script is None: | |||||
raise AssertionError( | |||||
"No script provided for the build {}".format( | |||||
self.name | |||||
) | |||||
) | |||||
# ... and that the script file can be executed | |||||
self.script_path = Path(self.script_root.joinpath(script)) | |||||
if not self.script_path.is_file() or not os.access(self.script_path, os.X_OK): | |||||
raise FileNotFoundError( | |||||
"The script file {} does not exist or does not have execution permission".format( | |||||
str(self.script_path) | |||||
) | |||||
) | |||||
# Create the build directory as needed | # Create the build directory as needed | ||||
self.build_directory = Path( | self.build_directory = Path( | ||||
self.project_root.joinpath( | self.project_root.joinpath( | ||||
'abc-ci-builds', | 'abc-ci-builds', | ||||
self.name)) | self.name)) | ||||
self.build_directory.mkdir(exist_ok=True, parents=True) | self.build_directory.mkdir(exist_ok=True, parents=True) | ||||
# Define the junit and logs directories | # Define the junit and logs directories | ||||
self.junit_reports_dir = self.build_directory.joinpath("test/junit") | self.junit_reports_dir = self.build_directory.joinpath("test/junit") | ||||
self.test_logs_dir = self.build_directory.joinpath("test/log") | self.test_logs_dir = self.build_directory.joinpath("test/log") | ||||
def create_build_steps(self, artifact_dir): | |||||
# There are 2 possibilities to define the build steps: | |||||
# - By defining a script to run. If such a script is set and is | |||||
# executable, it is the only thing to run. | |||||
# - By defining the configuration options and a list of target groups to | |||||
# run. The configuration step should be run once then all the targets | |||||
# groups. Each target group can contain 1 or more targets which | |||||
# should be run parallel. | |||||
script = self.config.get("script", None) | |||||
if script: | |||||
script_path = Path(self.script_root.joinpath(script)) | |||||
if not script_path.is_file() or not os.access(script_path, os.X_OK): | |||||
raise FileNotFoundError( | |||||
"The script file {} does not exist or does not have execution permission".format( | |||||
str(script_path) | |||||
) | |||||
) | |||||
self.build_steps = [ | |||||
{ | |||||
"bin": str(script_path), | |||||
"args": [], | |||||
} | |||||
] | |||||
return | |||||
# Get the cmake configuration definitions. | |||||
self.cmake_flags = self.config.get("cmake_flags", []) | |||||
self.cmake_flags.append("-DCMAKE_INSTALL_PREFIX={}".format( | |||||
str(artifact_dir))) | |||||
# Get the targets to build. If none is provided then raise an error. | |||||
targets = self.config.get("targets", None) | |||||
if not targets: | |||||
raise AssertionError( | |||||
"No build target has been provided for build {} and no script is defined, aborting".format( | |||||
self.name | |||||
) | |||||
) | |||||
# Some more flags for the build_cmake.sh script | |||||
build_cmake_flags = [] | |||||
if self.config.get("Werror", False): | |||||
build_cmake_flags.append("--Werror") | |||||
if self.config.get("junit", True): | |||||
build_cmake_flags.append("--junit") | |||||
if self.config.get("clang", False): | |||||
build_cmake_flags.append("--clang") | |||||
# Some generator flags | |||||
generator_flags = [] | |||||
if self.config.get("fail_fast", False): | |||||
generator_flags.append("-k0") | |||||
# First call should use the build_cmake.sh script in order to run | |||||
# cmake. | |||||
self.build_steps = [ | |||||
{ | |||||
"bin": str(self.project_root.joinpath("contrib/devtools/build_cmake.sh")), | |||||
"args": targets[0] + build_cmake_flags, | |||||
} | |||||
] | |||||
for target_group in targets[1:]: | |||||
self.build_steps.append( | |||||
{ | |||||
# TODO: let the generator be configurable | |||||
"bin": "ninja", | |||||
"args": generator_flags + target_group, | |||||
} | |||||
) | |||||
def get(self, key, default): | def get(self, key, default): | ||||
return self.config.get(key, default) | return self.config.get(key, default) | ||||
class UserBuild(): | class UserBuild(): | ||||
def __init__(self, configuration): | def __init__(self, configuration): | ||||
self.configuration = configuration | self.configuration = configuration | ||||
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | async def process_stdout(self, stdout): | ||||
self.print_line_to_logs(line) | self.print_line_to_logs(line) | ||||
except ValueError: | except ValueError: | ||||
self.print_line_to_logs( | self.print_line_to_logs( | ||||
"--- Line discarded due to StreamReader overflow ---" | "--- Line discarded due to StreamReader overflow ---" | ||||
) | ) | ||||
continue | continue | ||||
async def run_build(self, args=[]): | def run_process(self, bin, args=[]): | ||||
proc = await asyncio.create_subprocess_exec( | return asyncio.create_subprocess_exec( | ||||
*([str(self.configuration.script_path)] + args), | *([bin] + args), | ||||
# Buffer limit is 64KB by default, but we need a larger buffer: | # Buffer limit is 64KB by default, but we need a larger buffer: | ||||
limit=1024 * 256, | limit=1024 * 256, | ||||
stdout=asyncio.subprocess.PIPE, | stdout=asyncio.subprocess.PIPE, | ||||
stderr=asyncio.subprocess.STDOUT, | stderr=asyncio.subprocess.STDOUT, | ||||
cwd=self.configuration.build_directory, | cwd=self.configuration.build_directory, | ||||
env={ | env={ | ||||
**os.environ, | **os.environ, | ||||
**self.environment_variables, | **self.environment_variables, | ||||
**self.configuration.get("env", {}) | **self.configuration.get("env", {}), | ||||
"CMAKE_FLAGS": " ".join(self.configuration.cmake_flags), | |||||
}, | }, | ||||
) | ) | ||||
async def run_build(self, bin, args=[]): | |||||
proc = await self.run_process(bin, args) | |||||
await asyncio.wait([ | await asyncio.wait([ | ||||
self.process_stdout(proc.stdout) | self.process_stdout(proc.stdout) | ||||
]) | ]) | ||||
return await proc.wait() | return await proc.wait() | ||||
async def wait_for_build(self, timeout, args=[]): | async def wait_for_build(self, timeout, args=[]): | ||||
message = "Build {} completed successfully".format( | message = "Build {} completed successfully".format( | ||||
self.configuration.name | self.configuration.name | ||||
) | ) | ||||
try: | try: | ||||
return_code = await asyncio.wait_for(self.run_build(args), timeout) | for step in self.configuration.build_steps: | ||||
return_code = await asyncio.wait_for(self.run_build(step["bin"], step["args"]), timeout) | |||||
if return_code != 0: | if return_code != 0: | ||||
message = "Build {} failed with exit code {}".format( | message = "Build {} failed with exit code {}".format( | ||||
self.configuration.name, | self.configuration.name, | ||||
return_code | return_code | ||||
) | ) | ||||
return | |||||
except asyncio.TimeoutError: | except asyncio.TimeoutError: | ||||
message = "Build {} timed out after {:.1f}s".format( | message = "Build {} timed out after {:.1f}s".format( | ||||
self.configuration.name, round(timeout, 1) | self.configuration.name, round(timeout, 1) | ||||
) | ) | ||||
# The process is killed, set return code to 128 + 9 (SIGKILL) = 137 | # The process is killed, set return code to 128 + 9 (SIGKILL) = 137 | ||||
return_code = 137 | return_code = 137 | ||||
finally: | finally: | ||||
self.print_line_to_logs(message) | self.print_line_to_logs(message) | ||||
Show All 13 Lines | async def wait_for_build(self, timeout, args=[]): | ||||
return (return_code, message) | return (return_code, message) | ||||
def run(self, args=[]): | def run(self, args=[]): | ||||
if self.artifact_dir.is_dir(): | if self.artifact_dir.is_dir(): | ||||
shutil.rmtree(self.artifact_dir) | shutil.rmtree(self.artifact_dir) | ||||
self.artifact_dir.mkdir(exist_ok=True) | self.artifact_dir.mkdir(exist_ok=True) | ||||
self.configuration.create_build_steps(self.artifact_dir) | |||||
return_code, message = asyncio.run( | return_code, message = asyncio.run( | ||||
self.wait_for_build( | self.wait_for_build( | ||||
self.configuration.get( | self.configuration.get( | ||||
"timeout", DEFAULT_TIMEOUT)) | "timeout", DEFAULT_TIMEOUT)) | ||||
) | ) | ||||
return (return_code, message) | return (return_code, message) | ||||
▲ Show 20 Lines • Show All 105 Lines • Show Last 20 Lines |