diff --git a/contrib/buildbot/server.py b/contrib/buildbot/server.py index 580608213..3cae159de 100755 --- a/contrib/buildbot/server.py +++ b/contrib/buildbot/server.py @@ -1,1089 +1,1099 @@ #!/usr/bin/env python3 # # Copyright (c) 2019 The Bitcoin ABC developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. import hashlib import hmac import logging import os import re import shelve from functools import wraps from shlex import quote import yaml from build import BuildStatus, BuildTarget from deepmerge import always_merger from flask import Flask, abort, request from phabricator_wrapper import BITCOIN_ABC_PROJECT_PHID from shieldio import RasterBadge from teamcity_wrapper import TeamcityRequestException # Some keywords used by TeamCity and tcWebHook SUCCESS = "success" FAILURE = "failure" RUNNING = "running" UNRESOLVED = "UNRESOLVED" LANDBOT_BUILD_TYPE = "BitcoinAbcLandBot" # FIXME: figure out why the base64 logo started causing phabricator to # get a 502 response with the embedded {image} link. # In the meantime use the TeamCity icon from simpleicons.org # # with open(os.path.join(os.path.dirname(__file__), 'resources', 'teamcity-icon-16.base64'), 'rb') as icon: # BADGE_TC_BASE = RasterBadge( # label='TC build', # logo='data:image/png;base64,{}'.format( # icon.read().strip().decode('utf-8')), # ) BADGE_TC_BASE = RasterBadge( label='TC build', logo='TeamCity' ) BADGE_CIRRUS_BASE = RasterBadge( label='Cirrus build', logo='cirrus-ci' ) def create_server(tc, phab, slackbot, cirrus, db_file_no_ext=None, jsonEncoder=None): # Create Flask app for use as decorator app = Flask("abcbot") app.logger.setLevel(logging.INFO) # json_encoder can be overridden for testing if jsonEncoder: app.json_encoder = jsonEncoder phab.setLogger(app.logger) tc.set_logger(app.logger) cirrus.set_logger(app.logger) # Optionally persistable database create_server.db = { # A collection of the known build targets 'diff_targets': {}, # Build status panel data 'panel_data': {}, # Whether the last status check of master was green 'master_is_green': True, # Coverage panel data 'coverage_data': {}, } # If db_file_no_ext is not None, attempt to restore old database state if db_file_no_ext: app.logger.info( f"Loading persisted state database with base name '{db_file_no_ext}'...") try: with shelve.open(db_file_no_ext, flag='r') as db: for key in create_server.db.keys(): if key in db: create_server.db[key] = db[key] app.logger.info( f"Restored key '{key}' from persisted state") except BaseException: app.logger.info( f"Persisted state database with base name '{db_file_no_ext}' could not be opened. A new one will be created when written to.") app.logger.info("Done") else: app.logger.warning( "No database file specified. State will not be persisted.") def persistDatabase(fn): @wraps(fn) def decorated_function(*args, **kwargs): fn_ret = fn(*args, **kwargs) # Persist database after executed decorated function if db_file_no_ext: with shelve.open(db_file_no_ext) as db: for key in create_server.db.keys(): db[key] = create_server.db[key] app.logger.debug("Persisted current state") else: app.logger.debug( "No database file specified. Persisting state is being skipped.") return fn_ret return decorated_function # This decorator specifies an HMAC secret environment variable to use for verifying # requests for the given route. Currently, we're using Phabricator to trigger these # routes as webhooks, and a separate HMAC secret is required for each hook. # Phabricator does not support basic auth for webhooks, so HMAC must be # used instead. def verify_hmac(secret_env): def decorator(fn): @wraps(fn) def decorated_function(*args, **kwargs): secret = os.getenv(secret_env, None) if not secret: app.logger.info( f"Error: HMAC env variable '{secret_env}' does not exist") abort(401) data = request.get_data() digest = hmac.new( secret.encode(), data, hashlib.sha256).hexdigest() hmac_header = request.headers.get( 'X-Phabricator-Webhook-Signature') if not hmac_header: abort(401) if not hmac.compare_digest( digest.encode(), hmac_header.encode()): abort(401) return fn(*args, **kwargs) return decorated_function return decorator def get_json_request_data(request): if not request.is_json: abort(415, "Expected content-type is 'application/json'") return request.get_json() def get_master_build_configurations(): # Get the configuration from master config = yaml.safe_load(phab.get_file_content_from_master( "contrib/teamcity/build-configurations.yml")) # Get a list of the templates, if any templates = config.get("templates", {}) # Get a list of the builds build_configs = {} for build_name, v in config.get('builds', {}).items(): # Merge the templates template_config = {} template_names = v.get("templates", []) for template_name in template_names: # Raise an error if the template does not exist if template_name not in templates: raise AssertionError( f"Build {build_name} configuration inherits from " f"template {template_name}, but the template does not " "exist." ) always_merger.merge( template_config, templates.get(template_name)) # Retrieve the full build configuration by applying the templates build_configs[build_name] = always_merger.merge(template_config, v) return build_configs @app.route("/getCurrentUser", methods=['GET']) def getCurrentUser(): return request.authorization.username if request.authorization else None @app.route("/backportCheck", methods=['POST']) @verify_hmac('HMAC_BACKPORT_CHECK') def backportCheck(): data = get_json_request_data(request) revisionId = data['object']['phid'] revisionSearchArgs = { "constraints": { "phids": [revisionId], }, } data_list = phab.differential.revision.search( **revisionSearchArgs).data assert len(data_list) == 1, "differential.revision.search({}): Expected 1 revision, got: {}".format( revisionSearchArgs, data_list) summary = data_list[0]['fields']['summary'] foundPRs = 0 multilineCodeBlockDelimiters = 0 newSummary = "" for line in summary.splitlines(keepends=True): multilineCodeBlockDelimiters += len(re.findall(r'```', line)) # Only link PRs that do not reside in code blocks if multilineCodeBlockDelimiters % 2 == 0: def replacePRWithLink(baseUrl, prefix): def repl(match): nonlocal foundPRs # This check matches identation-based code blocks (2+ spaces) # and common cases for single-line code blocks (using # both single and triple backticks) if match.string.startswith(' ') or len( re.findall(r'`', match.string[:match.start()])) % 2 > 0: # String remains unchanged return match.group(0) else: # Backport PR is linked inline foundPRs += 1 PRNum = match.group(1) return f'[[{baseUrl}/{PRNum} | {prefix}#{PRNum}]]' return repl githubUrl = 'https://github.com/{}/pull' gitlabUrl = 'https://gitlab.com/{}/merge_requests' supportedRepos = { 'core': githubUrl.format('bitcoin/bitcoin'), 'core-gui': githubUrl.format('bitcoin-core/gui'), 'secp256k1': githubUrl.format('bitcoin-core/secp256k1'), 'bchn': gitlabUrl.format('bitcoin-cash-node/bitcoin-cash-node'), } for prefix, url in supportedRepos.items(): regEx = r'{}#(\d*)'.format(prefix) line = re.sub(regEx, replacePRWithLink( url, prefix), line) newSummary += line if foundPRs > 0: phab.updateRevisionSummary(revisionId, newSummary) return SUCCESS, 200 @app.route("/build", methods=['POST']) @persistDatabase def build(): buildTypeId = request.args.get('buildTypeId', None) ref = request.args.get('ref', 'refs/heads/master') PHID = request.args.get('PHID', None) abcBuildName = request.args.get('abcBuildName', None) properties = None if abcBuildName: properties = [{ 'name': 'env.ABC_BUILD_NAME', 'value': abcBuildName, }] build_id = tc.trigger_build(buildTypeId, ref, PHID, properties)['id'] if PHID in create_server.db['diff_targets']: build_target = create_server.db['diff_targets'][PHID] else: build_target = BuildTarget(PHID) build_target.queue_build(build_id, abcBuildName) create_server.db['diff_targets'][PHID] = build_target return SUCCESS, 200 @app.route("/buildDiff", methods=['POST']) @persistDatabase def build_diff(): def get_mandatory_argument(argument): value = request.args.get(argument, None) if value is None: raise AssertionError( "Calling /buildDiff endpoint with missing mandatory argument {}:\n{}".format( argument, request.args ) ) return value staging_ref = get_mandatory_argument('stagingRef') target_phid = get_mandatory_argument('targetPHID') revision_id = get_mandatory_argument('revisionId') # Get the list of changed files changedFiles = phab.get_revision_changed_files( revision_id=revision_id) build_configs = get_master_build_configurations() # Get a list of the builds that should run on diffs builds = [] for build_name, build_config in build_configs.items(): diffRegexes = build_config.get('runOnDiffRegex', None) if build_config.get('runOnDiff', False) or diffRegexes is not None: if diffRegexes: # If the regex matches at least one changed file, add this # build to the list. def regexesMatchAnyFile(regexes, files): for regex in regexes: for filename in files: if re.match(regex, filename): return True return False if regexesMatchAnyFile(diffRegexes, changedFiles): builds.append(build_name) else: builds.append(build_name) if target_phid in create_server.db['diff_targets']: build_target = create_server.db['diff_targets'][target_phid] else: build_target = BuildTarget(target_phid) for build_name in builds: properties = [{ 'name': 'env.ABC_BUILD_NAME', 'value': build_name, }, { 'name': 'env.ABC_REVISION', 'value': revision_id, }] build_id = tc.trigger_build( 'BitcoinABC_BitcoinAbcStaging', staging_ref, target_phid, properties)['id'] build_target.queue_build(build_id, build_name) if len(build_target.builds) > 0: create_server.db['diff_targets'][target_phid] = build_target else: phab.update_build_target_status(build_target) return SUCCESS, 200 @app.route("/land", methods=['POST']) def land(): data = get_json_request_data(request) revision = data['revision'] if not revision: return FAILURE, 400 # conduitToken is expected to be encrypted and will be decrypted by the # land bot. conduitToken = data['conduitToken'] if not conduitToken: return FAILURE, 400 committerName = data['committerName'] if not committerName: return FAILURE, 400 committerEmail = data['committerEmail'] if not committerEmail: return FAILURE, 400 properties = [{ 'name': 'env.ABC_REVISION', 'value': revision, }, { 'name': 'env.ABC_CONDUIT_TOKEN', 'value': conduitToken, }, { 'name': 'env.ABC_COMMITTER_NAME', 'value': committerName, }, { 'name': 'env.ABC_COMMITTER_EMAIL', 'value': committerEmail, }] output = tc.trigger_build( LANDBOT_BUILD_TYPE, 'refs/heads/master', UNRESOLVED, properties) if output: return output return FAILURE, 500 @app.route("/triggerCI", methods=['POST']) @verify_hmac('HMAC_TRIGGER_CI') def triggerCI(): data = get_json_request_data(request) app.logger.info(f"Received /triggerCI POST:\n{data}") # We expect a webhook with an edited object and a list of transactions. if "object" not in data or "transactions" not in data: return FAILURE, 400 data_object = data["object"] if "type" not in data_object or "phid" not in data_object: return FAILURE, 400 # We are searching for a specially crafted comment to trigger a CI # build. Only comments on revision should be parsed. Also if there is # no transaction, or the object is not what we expect, just return. if data_object["type"] != "DREV" or not data.get('transactions', []): return SUCCESS, 200 revision_PHID = data_object["phid"] # Retrieve the transactions details from their PHIDs transaction_PHIDs = [t["phid"] for t in data["transactions"] if "phid" in t] transactions = phab.transaction.search( objectIdentifier=revision_PHID, constraints={ "phids": transaction_PHIDs, } ).data # Extract the comments from the transaction list. Each transaction # contains a list of comments. comments = [c for t in transactions if t["type"] == "comment" for c in t["comments"]] # If there is no comment we have no interest in this webhook if not comments: return SUCCESS, 200 # Check if there is a specially crafted comment that should trigger a # CI build. Format: # @bot [build_name ...] def get_builds_from_comment(comment): tokens = comment.split() if not tokens or tokens.pop(0) != "@bot": return [] # Escape to prevent shell injection and remove duplicates return [quote(token) for token in list(set(tokens))] def next_token(current_token): next_token = { "": "PHID-TOKN-coin-1", "PHID-TOKN-coin-1": "PHID-TOKN-coin-2", "PHID-TOKN-coin-2": "PHID-TOKN-coin-3", "PHID-TOKN-coin-3": "PHID-TOKN-coin-4", "PHID-TOKN-coin-4": "PHID-TOKN-like-1", "PHID-TOKN-like-1": "PHID-TOKN-heart-1", "PHID-TOKN-heart-1": "PHID-TOKN-like-1", } return next_token[current_token] if current_token in next_token else "PHID-TOKN-like-1" def is_user_allowed_to_trigger_builds( user_PHID, current_token, comment_builds, build_configs): if current_token not in [ "", "PHID-TOKN-coin-1", "PHID-TOKN-coin-2", "PHID-TOKN-coin-3"]: return False if not all(role in phab.get_user_roles(user_PHID) for role in [ "verified", "approved", "activated", ]): return False for build_name in comment_builds: build_config = build_configs.get(build_name, None) if build_config is None: # If one of the build doesn't exist, reject them all. return False if "docker" in build_config: # If one of the build contain a docker configuration, reject # them all. return False return True # Anti DoS filter # # Users are allowed to trigger builds if these conditions are met: # - It is an ABC member # OR # | - It is a "verified", "approved" and "activated" user # | AND # | - The maximum number of requests for this revision has not been # | reached yet. # | AND # | - The build doesn't contain a `docker` configuration. # # The number of requests is tracked by awarding a coin token to the # revision each time a build request is submitted (the number of build # in that request is not taken into account). # The awarded coin token is graduated as follow: # "Haypence" => "Piece of Eight" => "Dubloon" => "Mountain of Wealth". # If the "Mountain of Wealth" token is reached, the next request will be # refused by the bot. At this stage only ABC members will be able to # trigger new builds. abc_members = phab.get_project_members(BITCOIN_ABC_PROJECT_PHID) current_token = phab.get_object_token(revision_PHID) build_configs = get_master_build_configurations() builds = [] for comment in comments: comment_builds = get_builds_from_comment(comment["content"]["raw"]) # Parsing the string is cheaper than phabricator requests, so check # if the comment is for us prior to filtering on the user. if not comment_builds: continue user = comment["authorPHID"] # ABC members can always trigger builds if user in abc_members: builds += comment_builds continue if is_user_allowed_to_trigger_builds( user, current_token, comment_builds, build_configs): builds += comment_builds # If there is no build provided, this request is not what we are after, # just return. # TODO return an help command to explain how to use the bot. if not builds: return SUCCESS, 200 # Give (only positive) feedback to user. If several comments are part of # the same transaction then there is no way to differentiate what the # token is for; however this is very unlikely to happen in real life. phab.set_object_token(revision_PHID, next_token(current_token)) staging_ref = phab.get_latest_diff_staging_ref(revision_PHID) # Trigger the requested builds for build in builds: # FIXME the hardcoded infos here should be gathered from somewhere tc.trigger_build( "BitcoinABC_BitcoinAbcStaging", staging_ref, properties=[{ 'name': 'env.ABC_BUILD_NAME', 'value': build, }] ) # If we reach this point, trigger_build did not raise an exception. return SUCCESS, 200 @app.route("/status", methods=['POST']) @persistDatabase def buildStatus(): out = get_json_request_data(request) app.logger.info(f"Received /status POST with data: {out}") return handle_build_result(**out) def send_harbormaster_build_link_if_required( build_link, build_target, build_name): # Check if a link to the build server has already been sent by searching # the artifacts. artifacts = phab.harbormaster.artifact.search( constraints={ "buildTargetPHIDs": [build_target.phid], } ).data build_link_artifact_key = build_name + "-" + build_target.phid # Search for the appropriated artifact key in the artifact list. # If found then the link is already set and there is no need to send it # again. for artifact in artifacts: if "artifactKey" in (artifact["fields"] or { }) and artifact["fields"]["artifactKey"] == build_link_artifact_key: return phab.harbormaster.createartifact( buildTargetPHID=build_target.phid, artifactKey=build_link_artifact_key, artifactType="uri", artifactData={ "uri": build_link, "name": build_name, "ui.external": True, } ) def update_build_status_panel(updated_build_type_id): # Perform a XOR like operation on the dicts: # - if a key from target is missing from reference, remove it from # target. # - if a key from reference is missing from target, add it to target. # The default value is the output of the default_value_callback(key). # - if the key exist in both, don't update it. # where target is a dictionary updated in place and reference a list of # keys. # Returns a tuple of (removed keys, added keys) def dict_xor(target, reference_keys, default_value_callback): removed_keys = [ k for k in list( target.keys()) if k not in reference_keys] for key in removed_keys: del target[key] added_keys = [ k for k in reference_keys if k not in list( target.keys())] for key in added_keys: target[key] = default_value_callback(key) return (removed_keys, added_keys) panel_content = '' def add_line_to_panel(line): return panel_content + line + '\n' def add_project_header_to_panel(project_name): return panel_content + ( '| {} | Status |\n' '|---|---|\n' ).format(project_name) # secp256k1 is a special case because it has a Cirrus build from a # Github repo that is not managed by the build-configurations.yml config. # The status always need to be fetched. sepc256k1_cirrus_status = cirrus.get_default_branch_status() cirrus_badge_url = BADGE_CIRRUS_BASE.get_badge_url( message=sepc256k1_cirrus_status.value, color=('brightgreen' if sepc256k1_cirrus_status == BuildStatus.Success else 'red' if sepc256k1_cirrus_status == BuildStatus.Failure else 'blue' if sepc256k1_cirrus_status == BuildStatus.Running else 'lightblue' if sepc256k1_cirrus_status == BuildStatus.Queued else 'inactive'), ) # Add secp256k1 Cirrus to the status panel. panel_content = add_project_header_to_panel( 'secp256k1 ([[https://github.com/Bitcoin-ABC/secp256k1 | Github]])') panel_content = add_line_to_panel( '| [[{} | {}]] | {{image uri="{}", alt="{}"}} |'.format( 'https://cirrus-ci.com/github/Bitcoin-ABC/secp256k1', 'master', cirrus_badge_url, sepc256k1_cirrus_status.value, ) ) panel_content = add_line_to_panel('') # Download the build configuration from master config = yaml.safe_load(phab.get_file_content_from_master( "contrib/teamcity/build-configurations.yml")) # Get a list of the builds to display config_build_names = [ k for k, v in config.get( 'builds', {}).items() if not v.get( 'hideOnStatusPanel', False)] # If there is no build to display, don't update the panel with teamcity # data if not config_build_names: phab.set_text_panel_content(17, panel_content) return # Associate with Teamcity data from the BitcoinABC project associated_builds = tc.associate_configuration_names( "BitcoinABC", config_build_names) # Get a unique list of the project ids project_ids = [build["teamcity_project_id"] for build in list(associated_builds.values())] project_ids = list(set(project_ids)) # Construct a dictionary from teamcity project id to project name. # This will make it easier to navigate from one to the other. project_name_map = {} for build in list(associated_builds.values()): project_name_map[build['teamcity_project_id'] ] = build['teamcity_project_name'] # If the list of project names has changed (project was added, deleted # or renamed, update the panel data accordingly. (removed_projects, added_projects) = dict_xor( create_server.db['panel_data'], project_ids, lambda key: {}) # Log the project changes if any if (len(removed_projects) + len(added_projects)) > 0: app.logger.info( "Teamcity project list has changed.\nRemoved: {}\nAdded: {}".format( removed_projects, added_projects, ) ) # Construct a dictionary from teamcity build type id to build name. # This will make it easier to navigate from one to the other. build_name_map = {} for build in list(associated_builds.values()): build_name_map[build['teamcity_build_type_id'] ] = build['teamcity_build_name'] def get_build_status_and_message(build_type_id): latest_build = tc.getLatestCompletedBuild(build_type_id) # If no build completed, set the status to unknown if not latest_build: build_status = BuildStatus.Unknown build_status_message = build_status.value else: build_info = tc.getBuildInfo(latest_build['id']) build_status = BuildStatus(build_info['status'].lower()) build_status_message = build_info.get( 'statusText', build_status.value) if build_status == BuildStatus.Failure else build_status.value return (build_status, build_status_message) # Update the builds for project_id, project_builds in sorted( create_server.db['panel_data'].items()): build_type_ids = [build['teamcity_build_type_id'] for build in list( associated_builds.values()) if build['teamcity_project_id'] == project_id] # If the list of builds has changed (build was added, deleted, # renamed, added to or removed from the items to display), update # the panel data accordingly. (removed_builds, added_builds) = dict_xor( project_builds, build_type_ids, # We need to fetch the satus for each added build lambda key: get_build_status_and_message(key) ) # Log the build changes if any if (len(removed_builds) + len(added_builds)) > 0: app.logger.info( "Teamcity build list has changed for project {}.\nRemoved: {}\nAdded: {}".format( project_id, removed_builds, added_builds, ) ) # From here only the build that triggered the call need to be # updated. Note that it might already be up-to-date if the build was # part of the added ones. # Other data remains valid from the previous calls. if updated_build_type_id not in added_builds and updated_build_type_id in list( project_builds.keys()): project_builds[updated_build_type_id] = get_build_status_and_message( updated_build_type_id) # Create a table view of the project: # # | | Status | # |------------------------------------| # | Link to latest build | Status icon | # | Link to latest build | Status icon | # | Link to latest build | Status icon | panel_content = add_project_header_to_panel( project_name_map[project_id]) for build_type_id, (build_status, build_status_message) in project_builds.items(): url = tc.build_url( "viewLog.html", { "buildTypeId": build_type_id, "buildId": "lastFinished" } ) # TODO insert Teamcity build failure message badge_url = BADGE_TC_BASE.get_badge_url( message=build_status_message, color=( 'lightgrey' if build_status == BuildStatus.Unknown else 'brightgreen' if build_status == BuildStatus.Success else 'red' ), ) panel_content = add_line_to_panel( '| [[{} | {}]] | {{image uri="{}", alt="{}"}} |'.format( url, build_name_map[build_type_id], badge_url, build_status_message, ) ) panel_content = add_line_to_panel('') phab.set_text_panel_content(17, panel_content) def update_coverage_panel(build_type_id, project_name, coverage_summary): coverage_permalink = "**[[ https://build.bitcoinabc.org/viewLog.html?buildId=lastSuccessful&buildTypeId={}&tab=report__Root_Code_Coverage&guest=1 | {} coverage report ]]**\n\n".format( build_type_id, project_name) coverage_report = "| Granularity | % hit | # hit | # total |\n" coverage_report += "| ----------- | ----- | ----- | ------- |\n" # Convert the textual coverage summary report to a pretty remarkup # content. # # The content loooks like this: # # Summary coverage rate: # lines......: 82.3% (91410 of 111040 lines) # functions..: 74.1% (6686 of 9020 functions) # branches...: 45.0% (188886 of 420030 branches) pattern = r"^\s*(?P\w+)\.+: (?P[0-9.]+%) \((?P\d+) of (?P\d+) .+$" for line in coverage_summary.splitlines(): match = re.match(pattern, line.strip()) if not match: continue coverage_report += "| {} | {} | {} | {} |\n".format( match.group('granularity').capitalize(), match.group('percent'), match.group('hit'), match.group('total'), ) # Cache the coverage data for this build type coverage_data = create_server.db['coverage_data'] coverage_data[build_type_id] = coverage_permalink + coverage_report # Update the coverage panel with our remarkup content phab.set_text_panel_content(21, "\n".join(coverage_data.values())) def handle_build_result(buildName, buildTypeId, buildResult, buildURL, branch, buildId, buildTargetPHID, projectName, **kwargs): # Do not report build status for ignored builds if phab.getIgnoreKeyword() in buildTypeId: return SUCCESS, 200 # Build didn't have a branch if branch == "UNRESOLVED": return FAILURE, 400 guest_url = tc.convert_to_guest_url(buildURL) status = BuildStatus(buildResult) isMaster = (branch == "refs/heads/master" or branch == "") # If a build completed on master, update the build status panel. if isMaster and ( status == BuildStatus.Success or status == BuildStatus.Failure): update_build_status_panel(buildTypeId) # If the build succeeded and there is a coverage report in the build # artifacts, update the coverage panel. if status == BuildStatus.Success: try: coverage_summary = tc.get_coverage_summary(buildId) except TeamcityRequestException: # The coverage report is not guaranteed to exist, in this # case teamcity will raise an exception. coverage_summary = None if coverage_summary: update_coverage_panel( buildTypeId, projectName, coverage_summary) # If we have a buildTargetPHID, report the status. build_target = create_server.db['diff_targets'].get( buildTargetPHID, None) if build_target is not None: phab.update_build_target_status(build_target, buildId, status) send_harbormaster_build_link_if_required( guest_url, build_target, build_target.builds[buildId].name ) if build_target.is_finished(): del create_server.db['diff_targets'][buildTargetPHID] revisionPHID = phab.get_revisionPHID(branch) buildInfo = tc.getBuildInfo(buildId) isAutomated = tc.checkBuildIsAutomated(buildInfo) if isAutomated and status == BuildStatus.Failure: # Check if this failure is infrastructure-related buildFailures = tc.getBuildProblems(buildId) if len(buildFailures) > 0: # If any infrastructure-related failures occurred, ping the right # people with a useful message. buildLog = tc.getBuildLog(buildId) if re.search(re.escape("[Infrastructure Error]"), buildLog): slackbot.postMessage('infra', " There was an infrastructure failure in '{}': {}".format( buildName, guest_url)) # Normally a comment of the build status is provided on diffs. Since no useful debug # info can be provided that is actionable to the user, we # give them a short message. if not isMaster: phab.commentOnRevision(revisionPHID, "(IMPORTANT) The build failed due to an unexpected infrastructure outage. " "The administrators have been notified to investigate. Sorry for the inconvenience.", buildName) return SUCCESS, 200 # Handle land bot builds if buildTypeId == LANDBOT_BUILD_TYPE: if status == BuildStatus.Success or status == BuildStatus.Failure: properties = buildInfo.getProperties() revisionId = properties.get( 'env.ABC_REVISION', 'MISSING REVISION ID') author = phab.getRevisionAuthor(revisionId) landBotMessage = "Failed to land your change:" if status == BuildStatus.Success: landBotMessage = "Successfully landed your change:" landBotMessage = "{}\nRevision: https://reviews.bitcoinabc.org/{}\nBuild: {}".format( landBotMessage, revisionId, guest_url) # Send a direct message to the revision author authorSlackUsername = phab.getAuthorSlackUsername(author) authorSlackUser = slackbot.getUserByName(authorSlackUsername) slackChannel = authorSlackUser['id'] if authorSlackUser else None if not slackChannel: slackChannel = 'dev' landBotMessage = "{}: Please set your slack username in your Phabricator profile so the landbot can send you direct messages: {}\n{}".format( authorSlackUsername, "https://reviews.bitcoinabc.org/people/editprofile/{}".format( author['id']), landBotMessage) slackbot.postMessage(slackChannel, landBotMessage) return SUCCESS, 200 # Open/update an associated task and message developers with relevant information if this build was # the latest completed, automated, master build of its type. if isMaster and isAutomated: latestBuild = tc.getLatestCompletedBuild(buildTypeId) latestBuildId = None if latestBuild: latestBuildId = latestBuild.get('id', None) logLatestBuildId = 'None' if latestBuildId is None else latestBuildId app.logger.info( "Latest completed build ID of type '{}': {}".format( buildTypeId, logLatestBuildId)) if latestBuildId == buildId: if status == BuildStatus.Success: updatedTask = phab.updateBrokenBuildTaskStatus( buildName, 'resolved') if updatedTask: # Only message once all of master is green (buildFailures, testFailures) = tc.getLatestBuildAndTestFailures( 'BitcoinABC') if len(buildFailures) == 0 and len(testFailures) == 0: if not create_server.db['master_is_green']: create_server.db['master_is_green'] = True slackbot.postMessage( 'dev', "Master is green again.") if status == BuildStatus.Failure: shortBuildUrl = tc.build_url( "viewLog.html", { "buildId": buildId, } ) # Explicitly ignored log lines. Use with care. buildLog = tc.getBuildLog(buildId) for line in tc.getIgnoreList(): # Skip empty lines and comments in the ignore file if not line or line.decode().strip()[0] == '#': continue # If any of the ignore patterns match any line in the # build log, ignore this failure if re.search(line.decode(), buildLog): return SUCCESS, 200 # Get number of build failures over the last few days numRecentFailures = tc.getNumAggregateFailuresSince( buildTypeId, 60 * 60 * 24 * 5) if numRecentFailures >= 3: # This build is likely flaky and the channel has # already been notified. return SUCCESS, 200 if numRecentFailures >= 2: # This build may be flaky. Ping the channel with a # less-noisy message. slackbot.postMessage('dev', f"Build '{buildName}' appears to be flaky: {shortBuildUrl}") return SUCCESS, 200 # Only mark master as red for failures that are not flaky create_server.db['master_is_green'] = False commitHashes = buildInfo.getCommits() newTask = phab.createBrokenBuildTask( buildName, guest_url, branch, commitHashes, 'rABC') if newTask: # TODO: Add 'Reviewed by: ' line # Do not point to a specific change for scheduled builds, as this generates noise for # the author of a change that is unlikely to contain # the root cause of the issue. if tc.checkBuildIsScheduled(buildInfo): slackbot.postMessage('dev', "Scheduled build '{}' appears to be broken: {}\n" "Task: https://reviews.bitcoinabc.org/T{}".format( buildName, shortBuildUrl, newTask['id'])) else: commitMap = phab.getRevisionPHIDsFromCommits( commitHashes) decoratedCommits = phab.decorateCommitMap( commitMap) decoratedCommit = decoratedCommits[commitHashes[0]] changeLink = decoratedCommit['link'] authorSlackUsername = decoratedCommit['authorSlackUsername'] authorSlackId = slackbot.formatMentionByName( authorSlackUsername) if not authorSlackId: authorSlackId = authorSlackUsername slackbot.postMessage('dev', "Committer: {}\n" "Build '{}' appears to be broken: {}\n" "Task: https://reviews.bitcoinabc.org/T{}\n" "Diff: {}".format( authorSlackId, buildName, shortBuildUrl, newTask['id'], changeLink)) if not isMaster: revisionId, authorPHID = phab.get_revision_info(revisionPHID) properties = buildInfo.getProperties() buildConfig = properties.get('env.ABC_BUILD_NAME', None) if not buildConfig: buildConfig = properties.get('env.OS_NAME', 'UNKNOWN') buildName = f"{buildName} ({buildConfig})" - if status == BuildStatus.Failure: + if status == BuildStatus.Success: + # Upon success, we only report if there is a website preview + # available. + preview_url_log = tc.getPreviewUrl(buildId) + if preview_url_log: + msg = phab.createBuildStatusMessage(status, guest_url, buildName) + msg += f"\n{preview_url_log}\n" + + phab.commentOnRevision(revisionPHID, msg, buildName) + + elif status == BuildStatus.Failure: msg = phab.createBuildStatusMessage( status, guest_url, buildName) # We add two newlines to break away from the (IMPORTANT) # callout. msg += '\n\n' testFailures = tc.getFailedTests(buildId) if len(testFailures) == 0: # If no test failure is available, print the tail of the # build log buildLog = tc.getBuildLog(buildId) logLines = [] for line in buildLog.splitlines(keepends=True): logLines.append(line) msg += "Tail of the build log:\n```lines=16,COUNTEREXAMPLE\n{}```".format( ''.join(logLines[-60:])) else: # Print the failure log for each test msg += 'Failed tests logs:\n' msg += '```lines=16,COUNTEREXAMPLE' for failure in testFailures: msg += "\n====== {} ======\n{}".format( failure['name'], failure['details']) msg += '```' msg += '\n\n' msg += 'Each failure log is accessible here:' for failure in testFailures: msg += f"\n[[{failure['logUrl']} | {failure['name']}]]" phab.commentOnRevision(revisionPHID, msg, buildName) return SUCCESS, 200 return app diff --git a/contrib/buildbot/teamcity_wrapper.py b/contrib/buildbot/teamcity_wrapper.py index 406cf5d22..b44e9dc51 100755 --- a/contrib/buildbot/teamcity_wrapper.py +++ b/contrib/buildbot/teamcity_wrapper.py @@ -1,489 +1,497 @@ #!/usr/bin/env python3 import io import json import os import re import time from collections import UserDict from pprint import pprint from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit from zipfile import ZipFile import requests class TeamcityRequestException(Exception): pass class BuildInfo(UserDict): @staticmethod def fromSingleBuildResponse(json_content): return BuildInfo(json_content['build'][0]) def getCommits(self): return [change['version'] for change in self.data['changes'] ['change']] if 'changes' in (self.data or {}) else None def getProperties(self): propsList = [] if 'properties' in (self.data or {}): propsList = self.data['properties']['property'] # Transform list of properties [{'name': name, 'value': value}, ...] into a # dict {name: value, ...} since we only care about the property values. properties = {} for prop in propsList: properties[prop['name']] = prop['value'] return properties if properties else None class TeamCity(): def __init__(self, base_url, username, password): self.session = requests.Session() self.base_url = base_url self.auth = (username, password) self.logger = None self.mockTime = None with open(os.path.join(os.path.dirname(__file__), 'ignore-logs.txt'), 'rb') as ignoreList: self.ignoreList = ignoreList.readlines() def set_logger(self, logger): self.logger = logger def getTime(self): if self.mockTime: return self.mockTime # time.time() returns a float, so we cast to an int to make it play nice with our other APIs. # We do not care about sub-second precision anyway. return int(time.time()) def getIgnoreList(self): return self.ignoreList def setMockTime(self, mockTime): self.mockTime = mockTime def getResponse(self, request, expectJson=True): response = self.session.send(request.prepare()) if response.status_code != requests.codes.ok: # Log the entire response, because something went wrong if self.logger: self.logger.info( "Request:\n{}\n\nResponse:\n{}".format( pprint( vars(request)), pprint( vars(response)))) raise TeamcityRequestException( f"Unexpected Teamcity API error! Status code: {response.status_code}") content = response.content if expectJson: content = json.loads(content) # Log the response content to aid in debugging if self.logger: self.logger.info(content) return content def trigger_build(self, buildTypeId, ref, PHID=None, properties=None): endpoint = self.build_url("app/rest/buildQueue") if not properties: properties = [] if PHID is not None: properties.append({ 'name': 'env.harborMasterTargetPHID', 'value': PHID, }) build = { 'branchName': ref, 'buildType': { 'id': buildTypeId }, 'properties': { 'property': properties, } } req = self._request('POST', endpoint, json.dumps(build)) return self.getResponse(req) def get_artifact(self, buildId, path): endpoint = self.build_url( f"app/rest/builds/id:{buildId}/artifacts/content/{path}" ) req = self._request('GET', endpoint) content = self.getResponse(req, expectJson=False) if not content: return None return content.decode('utf-8') def get_coverage_summary(self, buildId): return self.get_artifact( buildId, "coverage.tar.gz!/coverage-summary.txt") def get_clean_build_log(self, buildId): return self.get_artifact(buildId, "artifacts.tar.gz!/build.clean.log") def getBuildLog(self, buildId): # Try to get the clean build log first, then fallback to the full log try: clean_log = self.get_clean_build_log(buildId) if clean_log: return clean_log except TeamcityRequestException: # This is likely a 404 and the log doesn't exist. Either way, # ignore the failure since there is an alternative log we can # fetch. pass endpoint = self.build_url( "downloadBuildLog.html", { "buildId": buildId, "archived": "true", } ) req = self._request('GET', endpoint) content = self.getResponse(req, expectJson=False) ret = "" if not content: ret = "[Error Fetching Build Log]" else: z = ZipFile(io.BytesIO(content)) for filename in z.namelist(): for line in z.open(filename).readlines(): ret += line.decode('utf-8') return ret.replace('\r\n', '\n') + def getPreviewUrl(self, buildId): + try: + return self.get_artifact(buildId, "artifacts.tar.gz!/preview_url.log") + except TeamcityRequestException: + # This is likely a 404 and the log doesn't exist. + pass + return None + def getBuildProblems(self, buildId): endpoint = self.build_url( "app/rest/problemOccurrences", { "locator": f"build:(id:{buildId})", "fields": "problemOccurrence(id,details)", } ) req = self._request('GET', endpoint) content = self.getResponse(req) if 'problemOccurrence' in (content or {}): buildFailures = content['problemOccurrence'] for failure in buildFailures: # Note: Unlike test failures, build "problems" do not have # a well-defined focus line in the build log. For now, we # link to the footer to automatically scroll to the bottom # of the log where failures tend to be. failure['logUrl'] = self.build_url( "viewLog.html", { "tab": "buildLog", "logTab": "tree", "filter": "debug", "expand": "all", "buildId": buildId, }, "footer" ) return buildFailures return [] def getFailedTests(self, buildId): endpoint = self.build_url( "app/rest/testOccurrences", { "locator": f"build:(id:{buildId}),status:FAILURE", "fields": "testOccurrence(id,details,name)", } ) req = self._request('GET', endpoint) content = self.getResponse(req) if 'testOccurrence' in (content or {}): testFailures = content['testOccurrence'] for failure in testFailures: params = { "tab": "buildLog", "logTab": "tree", "filter": "debug", "expand": "all", "buildId": buildId, } match = re.search(r'id:(\d+)', failure['id']) if match: params['_focus'] = match.group(1) failure['logUrl'] = self.build_url( "viewLog.html", params ) return testFailures return [] def getBuildChangeDetails(self, changeId): endpoint = self.build_url(f"app/rest/changes/{changeId}") req = self._request('GET', endpoint) return self.getResponse(req) or {} def getBuildChanges(self, buildId): endpoint = self.build_url( "app/rest/changes", { "locator": f"build:(id:{buildId})", "fields": "change(id)" } ) req = self._request('GET', endpoint) content = self.getResponse(req) if 'change' in (content or {}): changes = content['change'] for i, change in enumerate(changes): changes[i] = self.getBuildChangeDetails(change['id']) return changes return [] def getBuildInfo(self, buildId): endpoint = self.build_url( "app/rest/builds", { "locator": f"id:{buildId}", # Note: Wildcard does not match recursively, so if you need data # from a sub-field, be sure to include it in the list. "fields": "build(*,changes(*),properties(*),triggered(*))", } ) req = self._request('GET', endpoint) content = self.getResponse(req) if 'build' in (content or {}): return BuildInfo.fromSingleBuildResponse(content) return BuildInfo() def checkBuildIsAutomated(self, buildInfo): trigger = buildInfo['triggered'] # Ignore builds by non-bot users, as these builds may be triggered for # any reason with various unknown configs return trigger['type'] != 'user' or trigger['user']['username'] == self.auth[0] def checkBuildIsScheduled(self, buildInfo): trigger = buildInfo['triggered'] return trigger['type'] == 'schedule' # For all nested build configurations under a project, fetch the latest # build failures. def getLatestBuildAndTestFailures(self, projectId): buildEndpoint = self.build_url( "app/rest/problemOccurrences", { "locator": f"currentlyFailing:true,affectedProject:(id:{projectId})", "fields": "problemOccurrence(*)", } ) buildReq = self._request('GET', buildEndpoint) buildContent = self.getResponse(buildReq) buildFailures = [] if 'problemOccurrence' in (buildContent or {}): buildFailures = buildContent['problemOccurrence'] testEndpoint = self.build_url( "app/rest/testOccurrences", { "locator": f"currentlyFailing:true,affectedProject:(id:{projectId})", "fields": "testOccurrence(*)", } ) testReq = self._request('GET', testEndpoint) testContent = self.getResponse(testReq) testFailures = [] if 'testOccurrence' in (testContent or {}): testFailures = testContent['testOccurrence'] return (buildFailures, testFailures) def getLatestCompletedBuild(self, buildType, build_fields=None): if not build_fields: build_fields = ['id'] endpoint = self.build_url( "app/rest/builds", { "locator": f"buildType:{buildType}", "fields": f"build({','.join(build_fields)})", "count": 1, } ) req = self._request('GET', endpoint) content = self.getResponse(req) builds = content.get('build', []) # There might be no build completed yet, in this case return None if not builds: return None # But there should be no more than a single build if len(builds) > 1: raise AssertionError( f"Unexpected Teamcity result. Called:\n{endpoint}\nGot:\n{content}" ) return builds[0] def formatTime(self, seconds): return time.strftime('%Y%m%dT%H%M%S%z', time.gmtime(seconds)) # The returned count is the number of groups of back-to-back failures, not # the number of individual failures def getNumAggregateFailuresSince(self, buildType, since): sinceTime = self.getTime() - since endpoint = self.build_url( "app/rest/builds", { "locator": f"buildType:{buildType},sinceDate:{self.formatTime(sinceTime)}", "fields": "build", } ) req = self._request('GET', endpoint) content = self.getResponse(req) if 'build' in (content or {}): builds = [{'status': 'SUCCESS'}] + content['build'] return sum([(builds[i - 1]['status'], builds[i]['status']) == ('SUCCESS', 'FAILURE') for i in range(1, len(builds))]) return 0 # For each of the given build name from the configuration file, associate the # teamcity build type id and teamcity build name def associate_configuration_names(self, project_id, config_names): # Get all the build configurations related to the given project, and # heavily filter the output to only return the id, name, project info # and the property name matching the configuration file. endpoint = self.build_url( "app/rest/buildTypes", { "locator": f"affectedProject:{project_id}", "fields": "buildType(project(id,name),id,name,parameters($locator(name:env.ABC_BUILD_NAME),property))", } ) req = self._request('GET', endpoint) content = self.getResponse(req) # Example of output: # "buildType": [ # { # "id": "BitcoinABC_Master_Build1", # "name": "My build 1", # "project": { # "id": "BitcoinABC_Master", # "name": "Master" # }, # "parameters": { # "property": [ # { # "name": "env.ABC_BUILD_NAME", # "value": "build-1" # } # ] # } # }, # { # "id": "BitcoinABC_Master_Build2", # "name": "My build 2", # "project": { # "id": "BitcoinABC_Master", # "name": "Master" # }, # "parameters": { # "property": [ # { # "name": "env.ABC_BUILD_NAME", # "value": "build-2" # } # ] # } # } # ] associated_config = {} for build_type in content.get('buildType', {}): if 'parameters' not in build_type: continue properties = build_type['parameters'].get('property', []) for build_property in properties: # Because of our filter, the only possible property is the one we # are after. Looking at the value is enough. config_name = build_property.get('value', None) if config_name in config_names: associated_config.update({ config_name: { "teamcity_build_type_id": build_type['id'], "teamcity_build_name": build_type['name'], "teamcity_project_id": build_type['project']['id'], "teamcity_project_name": build_type['project']['name'], } }) return associated_config def build_url(self, path="", params=None, fragment=None): if params is None: params = {} # Make guest access the default when not calling the rest API. # The caller can explicitly set guest=0 to bypass this behavior. if "guest" not in params and not path.startswith("app/rest/"): params["guest"] = 1 scheme, netloc = urlsplit(self.base_url)[0:2] return urlunsplit(( scheme, netloc, path, urlencode(params, doseq=True), fragment )) def convert_to_guest_url(self, url): parsed_url = urlsplit(url) # Don't touch unrelated URLs. parsed_base_url = urlsplit(self.base_url) if parsed_base_url.scheme != parsed_url.scheme or parsed_base_url.netloc != parsed_url.netloc: return url return self.build_url( parsed_url.path, parse_qs(parsed_url.query), parsed_url.fragment ) def _request(self, verb, url, data=None, headers=None): if self.logger: self.logger.info(f'{verb}: {url}') if headers is None: headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' } req = requests.Request( verb, url, auth=self.auth, headers=headers) req.data = data return req diff --git a/contrib/buildbot/test/test_endpoint_status.py b/contrib/buildbot/test/test_endpoint_status.py index e9b65865c..a3c69c6c7 100755 --- a/contrib/buildbot/test/test_endpoint_status.py +++ b/contrib/buildbot/test/test_endpoint_status.py @@ -1,1641 +1,1684 @@ #!/usr/bin/env python3 # # Copyright (c) 2017-2019 The Bitcoin ABC developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. import json import test.mocks.fixture import test.mocks.phabricator import test.mocks.teamcity import unittest from test.abcbot_fixture import ABCBotFixture from test.mocks.teamcity import DEFAULT_BUILD_ID, TEAMCITY_CI_USER from urllib.parse import urljoin import mock import requests +from build import BuildStatus from phabricator_wrapper import BITCOIN_ABC_REPO from server import BADGE_TC_BASE from teamcity_wrapper import BuildInfo from testutil import AnyWith -from build import BuildStatus - class statusRequestData(test.mocks.fixture.MockData): def __init__(self): self.buildName = 'build-name' self.projectName = 'bitcoin-abc-test' self.buildId = DEFAULT_BUILD_ID self.buildTypeId = 'build-type-id' self.buildResult = 'success' self.revision = 'commitHash' self.branch = 'refs/heads/master' self.buildTargetPHID = 'buildTargetPHID' def __setattr__(self, name, value): super().__setattr__(name, value) if name in ['buildId', 'buildTypeId']: self.buildURL = urljoin( test.mocks.teamcity.TEAMCITY_BASE_URL, 'viewLog.html?buildTypeId={}&buildId={}'.format( getattr(self, 'buildTypeId', ''), getattr(self, 'buildId', ''))) class EndpointStatusTestCase(ABCBotFixture): def setUp(self): super().setUp() self.phab.get_file_content_from_master = mock.Mock() self.phab.get_file_content_from_master.return_value = json.dumps({}) self.phab.set_text_panel_content = mock.Mock() self.teamcity.getBuildInfo = mock.Mock() self.configure_build_info() self.teamcity.get_coverage_summary = mock.Mock() self.teamcity.get_coverage_summary.return_value = None self.teamcity.getIgnoreList = mock.Mock() self.teamcity.getIgnoreList.return_value = [] self.cirrus.get_default_branch_status = mock.Mock() self.cirrus.get_default_branch_status.return_value = BuildStatus.Success def setup_master_failureAndTaskDoesNotExist(self, latestCompletedBuildId=DEFAULT_BUILD_ID, numRecentFailedBuilds=0, numCommits=1, userSearchFields=None, buildLogFile='testlog.zip'): if userSearchFields is None: userSearchFields = {} self.phab.maniphest.edit.return_value = { 'object': { 'id': '890', 'phid': 'PHID-TASK-890', }, } with open(self.data_dir / buildLogFile, 'rb') as f: buildLog = f.read() recentBuilds = [] if numRecentFailedBuilds == 0 else [ {'status': 'FAILURE'}, {'status': 'SUCCESS'}] * numRecentFailedBuilds self.teamcity.session.send.side_effect = [ # Build failures test.mocks.teamcity.Response(), # Latest completed build test.mocks.teamcity.Response(json.dumps({ 'build': [{ 'id': latestCompletedBuildId, }], })), test.mocks.teamcity.Response(status_code=requests.codes.not_found), test.mocks.teamcity.Response(buildLog), test.mocks.teamcity.Response(json.dumps({ 'build': recentBuilds, })), ] commits = [] for i in range(numCommits): commitId = 8000 + i commits.append({ 'phid': f'PHID-COMMIT-{commitId}', 'fields': { 'identifier': f'deadbeef0000011122233344455566677788{commitId}' }, }) self.phab.diffusion.commit.search.return_value = test.mocks.phabricator.Result( commits) revisionSearchResult = test.mocks.phabricator.differential_revision_search_result( total=numCommits) revisions = [] for i in range(numCommits): revisions.append({ 'sourcePHID': f'PHID-COMMIT-{8000 + i}', 'destinationPHID': revisionSearchResult.data[i]['phid'], }) self.phab.edge.search.return_value = test.mocks.phabricator.Result( revisions) self.phab.differential.revision.search.return_value = revisionSearchResult self.phab.user.search.return_value = test.mocks.phabricator.Result([{ 'id': '5678', 'phid': revisionSearchResult.data[0]['fields']['authorPHID'], 'fields': userSearchFields, }]) def configure_build_info(self, **kwargs): self.teamcity.getBuildInfo.return_value = BuildInfo.fromSingleBuildResponse( json.loads(test.mocks.teamcity.buildInfo(**kwargs).content) ) def test_status_invalid_json(self): data = "not: a valid json" response = self.app.post('/status', headers=self.headers, data=data) self.assertEqual(response.status_code, 415) def test_status_noData(self): response = self.app.post('/status', headers=self.headers) self.assertEqual(response.status_code, 415) self.phab.harbormaster.createartifact.assert_not_called() def test_status_unresolved(self): data = statusRequestData() data.branch = 'UNRESOLVED' data.buildTargetPHID = 'UNRESOLVED' response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 400) self.phab.harbormaster.createartifact.assert_not_called() def test_status_ignoredBuild(self): data = statusRequestData() data.buildTypeId = 'build-name__BOTIGNORE' response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.harbormaster.createartifact.assert_not_called() def test_status_master(self): data = statusRequestData() self.teamcity.session.send.side_effect = [ test.mocks.teamcity.buildInfo_automatedBuild(), test.mocks.teamcity.Response(json.dumps({ 'build': [{ 'id': DEFAULT_BUILD_ID, }], })), ] response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.phab.maniphest.edit.assert_not_called() self.slackbot.client.chat_postMessage.assert_not_called() def test_status_master_resolveBrokenBuildTask_masterGreen(self): def setupMockResponses(data): afterLatestBuild = [ test.mocks.teamcity.Response(), test.mocks.teamcity.Response(), ] if data.buildResult == 'failure': with open(self.data_dir / 'testlog.zip', 'rb') as f: buildLog = f.read() afterLatestBuild = [ test.mocks.teamcity.Response( status_code=requests.codes.not_found), test.mocks.teamcity.Response(buildLog), test.mocks.teamcity.Response(), ] self.teamcity.session.send.side_effect = [ test.mocks.teamcity.buildInfo_automatedBuild(), test.mocks.teamcity.Response(json.dumps({ 'build': [{ 'id': DEFAULT_BUILD_ID, }], })), ] + afterLatestBuild self.phab.maniphest.search.return_value = test.mocks.phabricator.Result([{ 'id': '123', 'phid': 'PHID-TASK-123', }]) self.phab.maniphest.edit.return_value = { 'object': { 'id': '123', 'phid': 'PHID-TASK-123', }, } data = statusRequestData() data.buildResult = 'failure' setupMockResponses(data) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) # Master should be marked red data = statusRequestData() setupMockResponses(data) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.phab.maniphest.edit.assert_called_with(transactions=[{ 'type': 'status', 'value': 'resolved', }], objectIdentifier='PHID-TASK-123') self.slackbot.client.chat_postMessage.assert_called_with( channel='#test-dev-channel', text="Master is green again.") def test_status_master_resolveBrokenBuildTask_masterStillRed(self): data = statusRequestData() self.configure_build_info( triggered=test.mocks.teamcity.buildInfo_triggered( triggerType='user', username=TEAMCITY_CI_USER) ) # Check build failure self.teamcity.session.send.side_effect = [ test.mocks.teamcity.Response(json.dumps({ 'build': [{ 'id': DEFAULT_BUILD_ID, }], })), test.mocks.teamcity.Response(json.dumps({ 'problemOccurrence': [{ 'build': { 'buildTypeId': 'build-type', }, }], })), test.mocks.teamcity.Response(), ] self.phab.maniphest.search.return_value = test.mocks.phabricator.Result([{ 'id': '123', 'phid': 'PHID-TASK-123', }]) self.phab.maniphest.edit.return_value = { 'object': { 'id': '123', 'phid': 'PHID-TASK-123', }, } response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.phab.maniphest.edit.assert_called_with(transactions=[{ 'type': 'status', 'value': 'resolved', }], objectIdentifier='PHID-TASK-123') self.slackbot.client.chat_postMessage.assert_not_called() # Check test failure self.teamcity.session.send.side_effect = [ test.mocks.teamcity.Response(json.dumps({ 'build': [{ 'id': DEFAULT_BUILD_ID, }], })), test.mocks.teamcity.Response(), test.mocks.teamcity.Response(json.dumps({ 'testOccurrence': [{ 'build': { 'buildTypeId': 'build-type', }, }], })), ] response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.phab.maniphest.edit.assert_called_with(transactions=[{ 'type': 'status', 'value': 'resolved', }], objectIdentifier='PHID-TASK-123') self.slackbot.client.chat_postMessage.assert_not_called() def test_status_master_resolveBrokenBuild_outOfOrderBuilds(self): data = statusRequestData() self.teamcity.session.send.side_effect = [ test.mocks.teamcity.buildInfo_automatedBuild(), test.mocks.teamcity.Response(json.dumps({ 'build': [{ # Another build of the same type that was started after this build # has already completed. Do not treat master as green/fixed based # on this build, since the most recent build may have # failed. 'id': 234567, }], })), ] response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.phab.maniphest.edit.assert_not_called() self.slackbot.client.chat_postMessage.assert_not_called() self.teamcity.session.send.assert_called_with(AnyWith(requests.PreparedRequest, { 'url': self.teamcity.build_url( "app/rest/builds", { "locator": "buildType:build-type-id", "fields": "build(id)", "count": 1, } ) })) def test_status_infraFailure(self): # Test an infra failure on master data = statusRequestData() data.buildResult = 'failure' with open(self.data_dir / 'testlog_infrafailure.zip', 'rb') as f: buildLog = f.read() def setupTeamcity(): self.configure_build_info( triggered=test.mocks.teamcity.buildInfo_triggered( triggerType='user', username=TEAMCITY_CI_USER) ) self.teamcity.session.send.side_effect = [ test.mocks.teamcity.Response(json.dumps({ 'problemOccurrence': [{ 'id': 'id:2500,build:(id:56789)', }], })), test.mocks.teamcity.Response( status_code=requests.codes.not_found), test.mocks.teamcity.Response(buildLog), ] setupTeamcity() response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.phab.maniphest.edit.assert_not_called() def verifyInfraChannelMessage(): self.slackbot.client.chat_postMessage.assert_called_with( channel='#infra-support-channel', text=" There was an infrastructure failure in 'build-name': {}".format( self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": DEFAULT_BUILD_ID, } ) )) verifyInfraChannelMessage() # Test an infra failure on a revision data = statusRequestData() data.branch = 'phabricator/diff/456' data.buildResult = 'failure' setupTeamcity() self.phab.differential.diff.search.return_value = test.mocks.phabricator.Result([{ 'id': '456', 'fields': { 'revisionPHID': '789' }, }]) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_called_with(transactions=[{ "type": "comment", "value": "(IMPORTANT) The build failed due to an unexpected infrastructure outage. " "The administrators have been notified to investigate. Sorry for the inconvenience.", }], objectIdentifier='789') self.phab.maniphest.edit.assert_not_called() verifyInfraChannelMessage() def test_status_master_failureAndTaskDoesNotExist_outOfOrderBuilds(self): data = statusRequestData() data.buildResult = 'failure' # Another build of the same type that was started after this build # has already completed. Do not treat master as red/broken based # on this build, since the most recent build may have succeeded. self.setup_master_failureAndTaskDoesNotExist( latestCompletedBuildId=234567) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.phab.maniphest.edit.assert_not_called() self.slackbot.client.chat_postMessage.assert_not_called() def test_status_master_failureAndTaskDoesNotExist_doNotIgnoreComments( self): data = statusRequestData() data.buildResult = 'failure' self.setup_master_failureAndTaskDoesNotExist(userSearchFields={ 'username': 'author-phab-username', 'custom.abc:slack-username': '', }) self.slackbot.client.users_list.return_value = test.mocks.slackbot.users_list( total=2) # Make sure comment patterns do not give false positives self.teamcity.getIgnoreList.return_value = [b'# TOTAL', b' # TOTAL'] response = self.app.post('/status', headers=self.headers, json=data) assert response.status_code == 200 self.phab.differential.revision.edit.assert_not_called() # Despite '# TOTAL' being in the build log, the failure was NOT ignored # since the ignore pattern is a comment. self.phab.maniphest.edit.assert_called() self.slackbot.client.chat_postMessage.assert_called() def test_status_master_failureAndTaskDoesNotExist_authorDefaultName(self): data = statusRequestData() data.buildResult = 'failure' self.setup_master_failureAndTaskDoesNotExist(userSearchFields={ 'username': 'author-phab-username', 'custom.abc:slack-username': '', }) self.slackbot.client.users_list.return_value = test.mocks.slackbot.users_list( total=2) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() maniphestEditCalls = [mock.call(transactions=[{ "type": "title", "value": "Build build-name is broken.", }, { "type": "priority", "value": "unbreak", }, { "type": "description", "value": "[[ {} | build-name ]] is broken on branch 'refs/heads/master'\n\nAssociated commits:\nrABCdeadbeef00000111222333444555666777888000".format( self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": DEFAULT_BUILD_ID, } ) ), }])] self.phab.maniphest.edit.assert_has_calls( maniphestEditCalls, any_order=False) self.phab.diffusion.commit.search.assert_called_with(constraints={ "repositories": [BITCOIN_ABC_REPO], "identifiers": ['deadbeef00000111222333444555666777888000'], }) self.phab.edge.search.assert_called_with( types=['commit.revision'], sourcePHIDs=['PHID-COMMIT-8000']) self.phab.differential.revision.search.assert_called_with( constraints={"phids": ['PHID-DREV-1000']}) self.slackbot.client.chat_postMessage.assert_called_with( channel='#test-dev-channel', text="Committer: author-phab-username\n" "Build 'build-name' appears to be broken: {}\n" "Task: https://reviews.bitcoinabc.org/T890\n" "Diff: https://reviews.bitcoinabc.org/D{}".format( self.teamcity.build_url( "viewLog.html", { "buildId": DEFAULT_BUILD_ID, } ), test.mocks.phabricator.DEFAULT_REVISION_ID, ) ) def test_status_master_failureAndTaskDoesNotExist_authorSlackUsername( self): data = statusRequestData() data.buildResult = 'failure' slackUserProfile = test.mocks.slackbot.userProfile( {'real_name': 'author-slack-username'}) slackUser = test.mocks.slackbot.user( userId='U8765', profile=slackUserProfile) self.setup_master_failureAndTaskDoesNotExist(userSearchFields={ 'username': 'author-phab-username', 'custom.abc:slack-username': 'author-slack-username', }) self.slackbot.client.users_list.return_value = test.mocks.slackbot.users_list( total=2, initialUsers=[slackUser]) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() maniphestEditCalls = [mock.call(transactions=[{ "type": "title", "value": "Build build-name is broken.", }, { "type": "priority", "value": "unbreak", }, { "type": "description", "value": "[[ {} | build-name ]] is broken on branch 'refs/heads/master'\n\nAssociated commits:\nrABCdeadbeef00000111222333444555666777888000".format( self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": DEFAULT_BUILD_ID, } ) ), }])] self.phab.maniphest.edit.assert_has_calls( maniphestEditCalls, any_order=False) self.phab.diffusion.commit.search.assert_called_with(constraints={ "repositories": [BITCOIN_ABC_REPO], "identifiers": ['deadbeef00000111222333444555666777888000'], }) self.phab.edge.search.assert_called_with( types=['commit.revision'], sourcePHIDs=['PHID-COMMIT-8000']) self.phab.differential.revision.search.assert_called_with( constraints={"phids": ['PHID-DREV-1000']}) self.slackbot.client.chat_postMessage.assert_called_with( channel='#test-dev-channel', text="Committer: <@U8765>\n" "Build 'build-name' appears to be broken: {}\n" "Task: https://reviews.bitcoinabc.org/T890\n" "Diff: https://reviews.bitcoinabc.org/D{}".format( self.teamcity.build_url( "viewLog.html", { "buildId": DEFAULT_BUILD_ID, } ), test.mocks.phabricator.DEFAULT_REVISION_ID, ) ) def test_status_master_failureAndTaskDoesNotExist_scheduledBuild(self): data = statusRequestData() data.buildResult = 'failure' self.configure_build_info( triggered=test.mocks.teamcity.buildInfo_triggered( triggerType='schedule') ) self.setup_master_failureAndTaskDoesNotExist() response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() maniphestEditCalls = [mock.call(transactions=[{ "type": "title", "value": "Build build-name is broken.", }, { "type": "priority", "value": "unbreak", }, { "type": "description", "value": "[[ {} | build-name ]] is broken on branch 'refs/heads/master'\n\nAssociated commits:\nrABCdeadbeef00000111222333444555666777888000".format( self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": DEFAULT_BUILD_ID, } ) ), }])] self.phab.maniphest.edit.assert_has_calls( maniphestEditCalls, any_order=False) self.slackbot.client.chat_postMessage.assert_called_with( channel='#test-dev-channel', text="Scheduled build 'build-name' appears to be broken: {}\n" "Task: https://reviews.bitcoinabc.org/T890".format( self.teamcity.build_url( "viewLog.html", { "buildId": DEFAULT_BUILD_ID, } ) )) def test_status_master_failureAndTaskDoesNotExist_multipleChanges(self): data = statusRequestData() data.buildResult = 'failure' self.configure_build_info( changes=test.mocks.teamcity.buildInfo_changes([ 'deadbeef00000111222333444555666777888000', 'deadbeef00000111222333444555666777888001']), triggered=test.mocks.teamcity.buildInfo_triggered(triggerType='user', username=test.mocks.teamcity.TEAMCITY_CI_USER), ) self.setup_master_failureAndTaskDoesNotExist( numCommits=2, userSearchFields={ 'username': 'author-phab-username', 'custom.abc:slack-username': '', }) self.slackbot.client.users_list.return_value = test.mocks.slackbot.users_list( total=2) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() maniphestEditCalls = [mock.call(transactions=[{ "type": "title", "value": "Build build-name is broken.", }, { "type": "priority", "value": "unbreak", }, { "type": "description", "value": "[[ {} | build-name ]] is broken on branch 'refs/heads/master'\n\nAssociated commits:\nrABCdeadbeef00000111222333444555666777888000\nrABCdeadbeef00000111222333444555666777888001".format( self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": DEFAULT_BUILD_ID, } ) ), }])] self.phab.maniphest.edit.assert_has_calls( maniphestEditCalls, any_order=False) self.phab.diffusion.commit.search.assert_called_with(constraints={ "repositories": [BITCOIN_ABC_REPO], "identifiers": ['deadbeef00000111222333444555666777888000', 'deadbeef00000111222333444555666777888001'], }) self.phab.edge.search.assert_called_with( types=['commit.revision'], sourcePHIDs=[ 'PHID-COMMIT-8000', 'PHID-COMMIT-8001']) self.phab.differential.revision.search.assert_called_with( constraints={"phids": ['PHID-DREV-1000', 'PHID-DREV-1001']}) self.slackbot.client.chat_postMessage.assert_called_with( channel='#test-dev-channel', text="Committer: author-phab-username\n" "Build 'build-name' appears to be broken: {}\n" "Task: https://reviews.bitcoinabc.org/T890\n" "Diff: https://reviews.bitcoinabc.org/D{}".format( self.teamcity.build_url( "viewLog.html", { "buildId": DEFAULT_BUILD_ID, } ), test.mocks.phabricator.DEFAULT_REVISION_ID, )) def test_status_master_failureAndTaskDoesNotExist_firstFlakyFailure(self): self.teamcity.setMockTime(1590000000) data = statusRequestData() data.buildResult = 'failure' self.setup_master_failureAndTaskDoesNotExist(numRecentFailedBuilds=2) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.teamcity.session.send.assert_called_with(AnyWith(requests.PreparedRequest, { 'url': self.teamcity.build_url( "app/rest/builds", { "locator": "buildType:{},sinceDate:{}".format(data.buildTypeId, self.teamcity.formatTime(1590000000 - 60 * 60 * 24 * 5)), "fields": "build", } ) })) self.phab.maniphest.edit.assert_not_called() self.slackbot.client.chat_postMessage.assert_called_with( channel='#test-dev-channel', text="Build 'build-name' appears to be flaky: {}".format( self.teamcity.build_url( "viewLog.html", { "buildId": DEFAULT_BUILD_ID, } )) ) def test_status_master_failureAndTaskDoesNotExist_successiveFlakyFailures( self): self.teamcity.setMockTime(1590000000) data = statusRequestData() data.buildResult = 'failure' self.setup_master_failureAndTaskDoesNotExist(numRecentFailedBuilds=3) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.teamcity.session.send.assert_called_with(AnyWith(requests.PreparedRequest, { 'url': self.teamcity.build_url( "app/rest/builds", { "locator": "buildType:{},sinceDate:{}".format(data.buildTypeId, self.teamcity.formatTime(1590000000 - 60 * 60 * 24 * 5)), "fields": "build", } ) })) self.phab.maniphest.edit.assert_not_called() self.slackbot.client.chat_postMessage.assert_not_called() def test_status_master_failureAndTaskDoesNotExist_ignoreFailure(self): testPatterns = [ # Simple match b'err:ntdll:RtlpWaitForCriticalSection', # Greedy match with some escaped characters br'\d*:err:ntdll:RtlpWaitForCriticalSection section .* retrying \(60 sec\)', # Less greedy match br'err:ntdll:RtlpWaitForCriticalSection section \w* "\?" wait timed out in thread \w*, blocked by \w*, retrying', ] for pattern in testPatterns: self.teamcity.getIgnoreList.return_value = [ b'# Some comment', b' # Another comment followed by an empty line', b'', pattern, ] data = statusRequestData() data.buildResult = 'failure' self.setup_master_failureAndTaskDoesNotExist( buildLogFile='testlog_ignore.zip') response = self.app.post( '/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.teamcity.session.send.assert_called_with(AnyWith(requests.PreparedRequest, { 'url': self.teamcity.build_url( "downloadBuildLog.html", { "buildId": DEFAULT_BUILD_ID, "archived": "true", "guest": 1, } ) })) self.phab.maniphest.edit.assert_not_called() self.slackbot.client.chat_postMessage.assert_not_called() def test_status_master_failureAndTaskExists(self): data = statusRequestData() data.buildResult = 'failure' with open(self.data_dir / 'testlog.zip', 'rb') as f: buildLog = f.read() self.teamcity.session.send.side_effect = [ test.mocks.teamcity.buildInfo_automatedBuild(), test.mocks.teamcity.Response(json.dumps({ 'build': [{ 'id': DEFAULT_BUILD_ID, }], })), test.mocks.teamcity.Response(status_code=requests.codes.not_found), test.mocks.teamcity.Response(buildLog), test.mocks.teamcity.Response(), ] self.phab.maniphest.search.return_value = test.mocks.phabricator.Result([{ 'id': '123', }]) response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_not_called() self.phab.maniphest.edit.assert_not_called() def test_status_revision_happyPath(self): data = statusRequestData() data.branch = 'phabricator/diff/456' self.configure_build_info( properties=test.mocks.teamcity.buildInfo_properties(propsList=[{ 'name': 'env.ABC_BUILD_NAME', 'value': 'build-diff', }]) ) self.phab.differential.revision.edit = mock.Mock() self.phab.differential.diff.search.return_value = test.mocks.phabricator.Result([{ 'id': '456', 'fields': { 'revisionPHID': '789' }, }]) self.phab.differential.revision.search.return_value = test.mocks.phabricator.differential_revision_search_result() response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) def test_status_revision_buildFailed(self): data = statusRequestData() data.buildResult = 'failure' data.branch = 'phabricator/diff/456' self.teamcity.getBuildLog = mock.Mock() self.teamcity.getBuildLog.return_value = "dummy log" self.configure_build_info( properties=test.mocks.teamcity.buildInfo_properties(propsList=[{ 'name': 'env.OS_NAME', 'value': 'linux', }]), ) self.teamcity.session.send.side_effect = [ test.mocks.teamcity.Response(), test.mocks.teamcity.Response(), test.mocks.teamcity.Response(), ] self.phab.differential.revision.edit = mock.Mock() self.phab.differential.diff.search.return_value = test.mocks.phabricator.Result([{ 'id': '456', 'fields': { 'revisionPHID': '789' }, }]) self.phab.differential.revision.search.return_value = test.mocks.phabricator.differential_revision_search_result() response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.edit.assert_called_with(transactions=[{ "type": "comment", "value": "(IMPORTANT) Build [[{} | build-name (linux)]] failed.\n\nTail of the build log:\n```lines=16,COUNTEREXAMPLE\ndummy log```".format( self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": DEFAULT_BUILD_ID, } ) ), }], objectIdentifier='789') + def test_status_preview_available(self): + data = statusRequestData() + data.buildResult = 'success' + data.branch = 'phabricator/diff/456' + + self.teamcity.getPreviewUrl = mock.Mock() + self.teamcity.getPreviewUrl.return_value = "Preview is available at http://127.0.0.1:8080 for the next 10 minutes." + + self.configure_build_info( + properties=test.mocks.teamcity.buildInfo_properties(propsList=[{ + 'name': 'env.OS_NAME', + 'value': 'linux', + }]), + ) + + self.teamcity.session.send.side_effect = [ + test.mocks.teamcity.Response(), + test.mocks.teamcity.Response(), + test.mocks.teamcity.Response(), + ] + + self.phab.differential.revision.edit = mock.Mock() + self.phab.differential.diff.search.return_value = test.mocks.phabricator.Result([{ + 'id': '456', + 'fields': { + 'revisionPHID': '789' + }, + }]) + self.phab.differential.revision.search.return_value = test.mocks.phabricator.differential_revision_search_result() + + response = self.app.post('/status', headers=self.headers, json=data) + self.assertEqual(response.status_code, 200) + build_url = self.teamcity.build_url( + "viewLog.html", + { + "buildTypeId": data.buildTypeId, + "buildId": DEFAULT_BUILD_ID, + } + ) + self.phab.differential.revision.edit.assert_called_with(transactions=[{ + "type": "comment", + "value": f"Build [[{build_url} | build-name (linux)]] passed.\nPreview is available at http://127.0.0.1:8080 for the next 10 minutes.\n", + }], objectIdentifier='789') + def test_status_revision_testsFailed(self): data = statusRequestData() data.branch = 'phabricator/diff/456' data.buildResult = 'failure' self.phab.differential.revision.edit = mock.Mock() self.phab.differential.diff.search.return_value = test.mocks.phabricator.Result([{ 'id': '456', 'fields': { 'revisionPHID': '789' }, }]) self.phab.differential.revision.search.return_value = test.mocks.phabricator.differential_revision_search_result() failures = [{ 'id': f'id:2500,build:(id:{DEFAULT_BUILD_ID})', 'details': 'stacktrace1', 'name': 'test name', }, { 'id': f'id:2620,build:(id:{DEFAULT_BUILD_ID})', 'details': 'stacktrace2', 'name': 'other test name', }] self.configure_build_info( properties=test.mocks.teamcity.buildInfo_properties(propsList=[{ 'name': 'env.ABC_BUILD_NAME', 'value': 'build-diff', }]) ) self.teamcity.session.send.side_effect = [ test.mocks.teamcity.Response(), test.mocks.teamcity.Response(json.dumps({ 'testOccurrence': failures, })) ] response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.teamcity.session.send.assert_called_with(AnyWith(requests.PreparedRequest, { 'url': self.teamcity.build_url( "app/rest/testOccurrences", { "locator": f"build:(id:{DEFAULT_BUILD_ID}),status:FAILURE", "fields": "testOccurrence(id,details,name)", } ) })) self.phab.differential.revision.edit.assert_called_with(transactions=[{ "type": "comment", "value": "(IMPORTANT) Build [[{} | build-name (build-diff)]] failed.\n\n" "Failed tests logs:\n" "```lines=16,COUNTEREXAMPLE" "\n====== test name ======\n" "stacktrace1" "\n====== other test name ======\n" "stacktrace2" "```" "\n\n" "Each failure log is accessible here:\n" "[[{} | test name]]\n" "[[{} | other test name]]".format( self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": DEFAULT_BUILD_ID, } ), self.teamcity.build_url( "viewLog.html", { "tab": "buildLog", "logTab": "tree", "filter": "debug", "expand": "all", "buildId": DEFAULT_BUILD_ID, "_focus": 2500, } ), self.teamcity.build_url( "viewLog.html", { "tab": "buildLog", "logTab": "tree", "filter": "debug", "expand": "all", "buildId": DEFAULT_BUILD_ID, "_focus": 2620, } ) ), }], objectIdentifier='789') def test_status_build_link_to_harbormaster(self): data = statusRequestData() data.buildTargetPHID = "PHID-HMBT-01234567890123456789" def call_build(build_id=DEFAULT_BUILD_ID, build_name=data.buildName): self.teamcity.session.send.side_effect = [ test.mocks.teamcity.buildInfo( build_id=build_id, buildqueue=True), ] url = 'build?buildTypeId=staging&ref=refs/tags/phabricator/diffs/{}&PHID={}&abcBuildName={}'.format( build_id, data.buildTargetPHID, build_name ) response = self.app.post(url, headers=self.headers) self.assertEqual(response.status_code, 200) # Set the status to 'running' to prevent target removal on completion. data.buildResult = "running" # Add some build target or there is no harbormaster build to link. call_build() def call_status_check_artifact_search(build_id=DEFAULT_BUILD_ID): self.teamcity.session.send.side_effect = [ test.mocks.teamcity.buildInfo_automatedBuild(), test.mocks.teamcity.buildInfo(build_id=build_id), ] response = self.app.post( '/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.harbormaster.artifact.search.assert_called_with( constraints={ "buildTargetPHIDs": ["PHID-HMBT-01234567890123456789"], } ) def check_createartifact(build_id=DEFAULT_BUILD_ID, build_name=data.buildName): self.phab.harbormaster.createartifact.assert_called_with( buildTargetPHID="PHID-HMBT-01234567890123456789", artifactKey=build_name + "-PHID-HMBT-01234567890123456789", artifactType="uri", artifactData={ "uri": self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": build_id, } ), "name": build_name, "ui.external": True, } ) # On first call the missing URL artifact should be added call_status_check_artifact_search() check_createartifact() # Furher calls to artifact.search will return our added URL artifact artifact_search_return_value = { "id": 123, "phid": "PHID-HMBA-abcdefghijklmnopqrst", "fields": { "buildTargetPHID": "PHID-HMBT-01234567890123456789", "artifactType": "uri", "artifactKey": data.buildName + "-PHID-HMBT-01234567890123456789", "isReleased": True, "dateCreated": 0, "dateModified": 0, "policy": {}, } } self.phab.harbormaster.artifact.search.return_value = test.mocks.phabricator.Result( [artifact_search_return_value]) # Reset the call counter self.phab.harbormaster.createartifact.reset_mock() # Call /status again a few time for i in range(10): call_status_check_artifact_search() # But since the artifact already exists it is not added again self.phab.harbormaster.createartifact.assert_not_called() # If the artifact key is not the expected one, the link is added artifact_search_return_value["fields"]["artifactKey"] = "unexpectedArtifactKey" self.phab.harbormaster.artifact.search.return_value = test.mocks.phabricator.Result( [artifact_search_return_value]) call_status_check_artifact_search() check_createartifact() # Add a few more builds to the build target for i in range(1, 1 + 5): build_id = DEFAULT_BUILD_ID + i build_name = f'build-{i}' data.buildName = build_name data.buildId = build_id data.buildTypeId = data.buildTypeId call_build(build_id, build_name) # Check the artifact is searched and add for each build call_status_check_artifact_search(build_id) check_createartifact(build_id, build_name) def test_status_landbot(self): data = statusRequestData() data.buildTypeId = 'BitcoinAbcLandBot' # Side effects are only valid once per call, so we need to set side_effect # for every call to the endpoint. def setupTeamcity(): self.configure_build_info( properties=test.mocks.teamcity.buildInfo_properties( propsList=[{ 'name': 'env.ABC_REVISION', 'value': 'D1234', }] ) ) self.teamcity.session.send.side_effect = [ test.mocks.teamcity.Response(), ] def setupUserSearch(slackUsername): self.phab.user.search.return_value = test.mocks.phabricator.Result([{ 'id': '5678', 'phid': revisionSearchResult.data[0]['fields']['authorPHID'], 'fields': { 'username': 'author-phab-username', 'custom.abc:slack-username': slackUsername, }, }]) slackUserProfile = test.mocks.slackbot.userProfile( {'real_name': 'author-slack-username'}) slackUser = test.mocks.slackbot.user( userId='U8765', profile=slackUserProfile) self.slackbot.client.users_list.return_value = test.mocks.slackbot.users_list( total=2, initialUsers=[slackUser]) revisionSearchResult = test.mocks.phabricator.differential_revision_search_result() self.phab.differential.revision.search.return_value = revisionSearchResult expectedBuildUrl = self.teamcity.build_url( "viewLog.html", { "buildTypeId": data.buildTypeId, "buildId": DEFAULT_BUILD_ID, } ) # Test happy path setupTeamcity() setupUserSearch(slackUsername='author-slack-username') response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.search.assert_called_with(constraints={ 'ids': [1234]}) self.phab.user.search.assert_called_with( constraints={'phids': [revisionSearchResult.data[0]['fields']['authorPHID']]}) self.slackbot.client.chat_postMessage.assert_called_with( channel='U8765', text="Successfully landed your change:\n" "Revision: https://reviews.bitcoinabc.org/D1234\n" "Build: {}".format(expectedBuildUrl), ) # Test direct message on a landbot build failure data.buildResult = 'failure' setupTeamcity() setupUserSearch(slackUsername='author-slack-username') response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.search.assert_called_with(constraints={ 'ids': [1234]}) self.phab.user.search.assert_called_with( constraints={'phids': [revisionSearchResult.data[0]['fields']['authorPHID']]}) self.slackbot.client.chat_postMessage.assert_called_with( channel='U8765', text="Failed to land your change:\n" "Revision: https://reviews.bitcoinabc.org/D1234\n" "Build: {}".format(expectedBuildUrl), ) # Test message on build failure when the author's slack username is # unknown setupUserSearch(slackUsername='') setupTeamcity() response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.search.assert_called_with(constraints={ 'ids': [1234]}) self.phab.user.search.assert_called_with( constraints={'phids': [revisionSearchResult.data[0]['fields']['authorPHID']]}) self.slackbot.client.chat_postMessage.assert_called_with( channel='#test-dev-channel', text="author-phab-username: Please set your slack username in your Phabricator profile so the " "landbot can send you direct messages: https://reviews.bitcoinabc.org/people/editprofile/5678\n" "Failed to land your change:\n" "Revision: https://reviews.bitcoinabc.org/D1234\n" "Build: {}".format(expectedBuildUrl), ) # Make sure no messages are sent on running status data.buildResult = 'running' setupTeamcity() self.phab.differential.revision.search = mock.Mock() self.phab.user.search = mock.Mock() self.slackbot.client.chat_postMessage = mock.Mock() response = self.app.post('/status', headers=self.headers, json=data) self.assertEqual(response.status_code, 200) self.phab.differential.revision.search.assert_not_called() self.phab.user.search.assert_not_called() self.slackbot.client.chat_postMessage.assert_not_called() def test_update_build_status_panel(self): panel_id = 17 self.phab.get_file_content_from_master = mock.Mock() self.phab.set_text_panel_content = mock.Mock() associated_builds = {} self.teamcity.associate_configuration_names = mock.Mock() self.teamcity.associate_configuration_names.return_value = associated_builds # List of failing build types failing_build_type_ids = [] # List of builds that did not complete yet no_complete_build_type_ids = [] # Builds with ID 42 are failures self.teamcity.getLatestCompletedBuild = mock.Mock() self.teamcity.getLatestCompletedBuild.side_effect = lambda build_type_id: ( {'id': 42} if build_type_id in failing_build_type_ids else None if build_type_id in no_complete_build_type_ids else {'id': DEFAULT_BUILD_ID} ) build_info = BuildInfo.fromSingleBuildResponse( json.loads(test.mocks.teamcity.buildInfo().content) ) def _get_build_info(build_id): status = BuildStatus.Failure if build_id == 42 else BuildStatus.Success build_info['id'] = build_id build_info['status'] = status.value.upper() build_info['statusText'] = "Build success" if status == BuildStatus.Success else "Build failure" return build_info self.teamcity.getBuildInfo.side_effect = _get_build_info def get_cirrus_panel_content(status=None): if not status: status = BuildStatus.Success return ( '| secp256k1 ([[https://github.com/Bitcoin-ABC/secp256k1 | Github]]) | Status |\n' '|---|---|\n' '| [[https://cirrus-ci.com/github/Bitcoin-ABC/secp256k1 | master]] | {{image uri="https://raster.shields.io/static/v1?label=Cirrus build&message={}&color={}&logo=cirrus-ci", alt="{}"}} |\n\n' ).format( status.value, 'brightgreen' if status == BuildStatus.Success else 'red', status.value, ) def set_config_file(names_to_display, names_to_hide): config = {"builds": {}} builds = config["builds"] for build_name in names_to_display: builds[build_name] = {"hideOnStatusPanel": False} for build_name in names_to_hide: builds[build_name] = {"hideOnStatusPanel": True} self.phab.get_file_content_from_master.return_value = json.dumps( config) def associate_build(name, teamcity_build_type_id=None, teamcity_build_name=None, teamcity_project_id=None, teamcity_project_name=None): if not teamcity_build_type_id: teamcity_build_type_id = f"{name}_Type" if not teamcity_build_name: teamcity_build_name = f"My Build {name}" if not teamcity_project_id: teamcity_project_id = "ProjectId" if not teamcity_project_name: teamcity_project_name = "Project Name" associated_builds[name] = { "teamcity_build_type_id": teamcity_build_type_id, "teamcity_build_name": teamcity_build_name, "teamcity_project_id": teamcity_project_id, "teamcity_project_name": teamcity_project_name, } self.teamcity.associate_configuration_names.return_value = associated_builds def call_status(build_type_id, status, branch=None, expected_status_code=None): data = statusRequestData() data.buildResult = status.value data.buildTypeId = build_type_id if branch: data.branch = branch response = self.app.post( '/status', headers=self.headers, json=data) self.assertEqual( response.status_code, 200 if not expected_status_code else expected_status_code) def assert_panel_content(content): self.phab.set_text_panel_content.assert_called_with( panel_id, content ) def header(project_name): header = f'| {project_name} | Status |\n' header += '|---|---|\n' return header def build_line(build_name, status=None, build_type_id=None, teamcity_build_name=None): if not status: status = BuildStatus.Success if not build_type_id: build_type_id = f"{build_name}_Type" if not teamcity_build_name: teamcity_build_name = f"My Build {build_name}" url = self.teamcity.build_url( "viewLog.html", { "buildTypeId": build_type_id, "buildId": "lastFinished" } ) status_message = "Build failure" if status == BuildStatus.Failure else status.value badge_url = BADGE_TC_BASE.get_badge_url( message=status_message, color=( 'lightgrey' if status == BuildStatus.Unknown else 'brightgreen' if status == BuildStatus.Success else 'red' ), ) return '| [[{} | {}]] | {{image uri="{}", alt="{}"}} |\n'.format( url, teamcity_build_name, badge_url, status_message, ) # No build in config file, should bail out and not edit the panel with # teamcity content set_config_file([], []) call_status('dont_care', BuildStatus.Success) assert_panel_content(get_cirrus_panel_content()) # If branch is not master the panel is not updated self.phab.set_text_panel_content.reset_mock() call_status( 'dont_care', BuildStatus.Success, branch='refs/tags/phabricator/diff/42', expected_status_code=500 ) self.phab.set_text_panel_content.assert_not_called() # Turn cirrus build into failure self.cirrus.get_default_branch_status.return_value = BuildStatus.Failure call_status('dont_care', BuildStatus.Success) assert_panel_content(get_cirrus_panel_content(BuildStatus.Failure)) self.cirrus.get_default_branch_status.return_value = BuildStatus.Success # Some builds in config file but no associated teamcity build set_config_file(["show_me11"], []) call_status('dont_care', BuildStatus.Success) assert_panel_content(get_cirrus_panel_content()) # Set one build to be shown and associate it. This is not the build that # just finished. associate_build("show_me11") call_status('hide_me_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name') + build_line('show_me11') + '\n' ) # Now with 3 builds from the same project + 1 not shown set_config_file(["show_me11", "show_me12", "show_me13"], ["hidden"]) associate_build("show_me12") associate_build("show_me13") call_status('hide_me_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name') + build_line('show_me11') + build_line('show_me12') + build_line('show_me13') + '\n' ) # Add 2 more builds from another project. # Check the result is always the same after a few calls set_config_file(["show_me11", "show_me12", "show_me13", "show_me21", "show_me22"], []) associate_build( "show_me21", teamcity_project_id="ProjectId2", teamcity_project_name="Project Name 2") associate_build( "show_me22", teamcity_project_id="ProjectId2", teamcity_project_name="Project Name 2") for i in range(10): call_status('hide_me_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name') + build_line('show_me11') + build_line('show_me12') + build_line('show_me13') + '\n' + header('Project Name 2') + build_line('show_me21') + build_line('show_me22') + '\n' ) # Remove a build from teamcity, but not from the config file del associated_builds["show_me12"] call_status('hide_me_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name') + build_line('show_me11') + build_line('show_me13') + '\n' + header('Project Name 2') + build_line('show_me21') + build_line('show_me22') + '\n' ) # Hide a build from the config file (cannot be associated anymore) set_config_file(["show_me11", "show_me12", "show_me21", "show_me22"], ["show_me13"]) del associated_builds["show_me13"] call_status('hide_me_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name') + build_line('show_me11') + '\n' + header('Project Name 2') + build_line('show_me21') + build_line('show_me22') + '\n' ) # Remove the last build from a project and check the project is no # longer shown del associated_builds["show_me11"] call_status('hide_me_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name 2') + build_line('show_me21') + build_line('show_me22') + '\n' ) # Check the status of the build is not checked if it didn't finish # through the endpoint failing_build_type_ids = ['show_me21_Type'] call_status('hide_me_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name 2') + build_line('show_me21') + build_line('show_me22') + '\n' ) # But having the build to be updated through the endpoint causes the # status to be fetched again. Note that the result is meaningless here, # and will be fetched from Teamcity anyway. call_status('show_me21_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name 2') + build_line('show_me21', status=BuildStatus.Failure) + build_line('show_me22') + '\n' ) # Check the unknown status of a build if it never completed associate_build( "show_me23", teamcity_project_id="ProjectId2", teamcity_project_name="Project Name 2") no_complete_build_type_ids = ['show_me23_Type'] call_status('show_me21_Type', BuildStatus.Success) assert_panel_content( get_cirrus_panel_content() + header('Project Name 2') + build_line('show_me21', status=BuildStatus.Failure) + build_line('show_me22') + build_line('show_me23', status=BuildStatus.Unknown) + '\n' ) def test_update_coverage_panel(self): panel_id = 21 buildTypeId = 'DummyBuildType' projectName = 'Dummy Project' self.phab.set_text_panel_content = mock.Mock() self.teamcity.get_coverage_summary.return_value = "Dummy" def call_status(status, expected_status_code=None): data = statusRequestData() data.buildResult = status.value data.buildTypeId = buildTypeId data.projectName = projectName response = self.app.post( '/status', headers=self.headers, json=data) self.assertEqual( response.status_code, 200 if not expected_status_code else expected_status_code) def assert_panel_content(content): self.phab.set_text_panel_content.assert_called_with( panel_id, content ) def _assert_not_called_with(self, *args, **kwargs): try: self.assert_called_with(*args, **kwargs) except AssertionError: return raise AssertionError( 'Expected {} to not have been called.'.format( self._format_mock_call_signature( args, kwargs))) mock.Mock.assert_not_called_with = _assert_not_called_with # Build failed: ignore call_status(BuildStatus.Failure, expected_status_code=500) self.phab.set_text_panel_content.assert_not_called_with( panel_id=panel_id) # No coverage report artifact: ignore self.teamcity.get_coverage_summary.return_value = None call_status(BuildStatus.Success, expected_status_code=500) self.phab.set_text_panel_content.assert_not_called_with( panel_id=panel_id) # Generate coverage report for one project self.teamcity.get_coverage_summary.return_value = \ """ Reading tracefile check-extended_combined.info Summary coverage rate: lines......: 82.3% (91410 of 111040 lines) functions..: 74.1% (6686 of 9020 functions) branches...: 45.0% (188886 of 420030 branches) """ call_status(BuildStatus.Success, expected_status_code=500) assert_panel_content( """**[[ https://build.bitcoinabc.org/viewLog.html?buildId=lastSuccessful&buildTypeId=DummyBuildType&tab=report__Root_Code_Coverage&guest=1 | Dummy Project coverage report ]]** | Granularity | % hit | # hit | # total | | ----------- | ----- | ----- | ------- | | Lines | 82.3% | 91410 | 111040 | | Functions | 74.1% | 6686 | 9020 | | Branches | 45.0% | 188886 | 420030 | """ ) # Generate coverage report for another project buildTypeId = 'AnotherBuildType' projectName = 'Another Project' self.teamcity.get_coverage_summary.return_value = \ """ Reading tracefile coverage/lcov.info Summary coverage rate: lines......: 20.0% (261 of 1305 lines) functions..: 16.9% (41 of 242 functions) branches...: 18.2% (123 of 676 branches) """ call_status(BuildStatus.Success, expected_status_code=500) assert_panel_content( """**[[ https://build.bitcoinabc.org/viewLog.html?buildId=lastSuccessful&buildTypeId=DummyBuildType&tab=report__Root_Code_Coverage&guest=1 | Dummy Project coverage report ]]** | Granularity | % hit | # hit | # total | | ----------- | ----- | ----- | ------- | | Lines | 82.3% | 91410 | 111040 | | Functions | 74.1% | 6686 | 9020 | | Branches | 45.0% | 188886 | 420030 | **[[ https://build.bitcoinabc.org/viewLog.html?buildId=lastSuccessful&buildTypeId=AnotherBuildType&tab=report__Root_Code_Coverage&guest=1 | Another Project coverage report ]]** | Granularity | % hit | # hit | # total | | ----------- | ----- | ----- | ------- | | Lines | 20.0% | 261 | 1305 | | Functions | 16.9% | 41 | 242 | | Branches | 18.2% | 123 | 676 | """ ) # Update one of the existing coverage reports buildTypeId = 'DummyBuildType' projectName = 'Renamed Dummy Project' self.teamcity.get_coverage_summary.return_value = \ """ Reading tracefile check-extended_combined.info Summary coverage rate: lines......: 82.4% (91411 of 111030 lines) functions..: 74.2% (6687 of 9010 functions) branches...: 45.1% (188887 of 420020 branches) """ call_status(BuildStatus.Success, expected_status_code=500) assert_panel_content( """**[[ https://build.bitcoinabc.org/viewLog.html?buildId=lastSuccessful&buildTypeId=DummyBuildType&tab=report__Root_Code_Coverage&guest=1 | Renamed Dummy Project coverage report ]]** | Granularity | % hit | # hit | # total | | ----------- | ----- | ----- | ------- | | Lines | 82.4% | 91411 | 111030 | | Functions | 74.2% | 6687 | 9010 | | Branches | 45.1% | 188887 | 420020 | **[[ https://build.bitcoinabc.org/viewLog.html?buildId=lastSuccessful&buildTypeId=AnotherBuildType&tab=report__Root_Code_Coverage&guest=1 | Another Project coverage report ]]** | Granularity | % hit | # hit | # total | | ----------- | ----- | ----- | ------- | | Lines | 20.0% | 261 | 1305 | | Functions | 16.9% | 41 | 242 | | Branches | 18.2% | 123 | 676 | """ ) if __name__ == '__main__': unittest.main()