Page MenuHomePhabricator
Differential D7994 Diff 24810 contrib/buildbot/server.py

Changeset View

Standalone View

View Options

contrib/buildbot/server.py

#!/usr/bin/env python3 #!/usr/bin/env python3
# #
# Copyright (c) 2019 The Bitcoin ABC developers # Copyright (c) 2019 The Bitcoin ABC developers
# Distributed under the MIT software license, see the accompanying # Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
from build import BuildStatus, BuildTarget from build import BuildStatus, BuildTarget
from flask import abort, Flask, request from flask import abort, Flask, request
from functools import wraps from functools import wraps
import hashlib import hashlib
import hmac import hmac
import logging
import os import os
from phabricator_wrapper import ( from phabricator_wrapper import (
BITCOIN_ABC_PROJECT_PHID, BITCOIN_ABC_PROJECT_PHID,
) )
import re import re
import shelve
from shieldio import RasterBadge from shieldio import RasterBadge
from shlex import quote from shlex import quote
from teamcity_wrapper import TeamcityRequestException from teamcity_wrapper import TeamcityRequestException
import yaml import yaml
# Some keywords used by TeamCity and tcWebHook # Some keywords used by TeamCity and tcWebHook
SUCCESS = "success" SUCCESS = "success"
Show All 12 Lineswith open(os.path.join(os.path.dirname(__file__), 'resources', 'teamcity-icon-16.base64'), 'rb') as icon:
) )
BADGE_TRAVIS_BASE = RasterBadge( BADGE_TRAVIS_BASE = RasterBadge(
label='Travis build', label='Travis build',
logo='travis' logo='travis'
) )
def create_server(tc, phab, slackbot, travis, jsonEncoder=None): def create_server(tc, phab, slackbot, travis,
db_file_no_ext=None, jsonEncoder=None):
# Create Flask app for use as decorator # Create Flask app for use as decorator
app = Flask("abcbot") app = Flask("abcbot")
app.logger.setLevel(logging.INFO)
# json_encoder can be overridden for testing # json_encoder can be overridden for testing
if jsonEncoder: if jsonEncoder:
app.json_encoder = jsonEncoder app.json_encoder = jsonEncoder
phab.setLogger(app.logger) phab.setLogger(app.logger)
tc.set_logger(app.logger) tc.set_logger(app.logger)
travis.set_logger(app.logger) travis.set_logger(app.logger)
# Optionally persistable database
create_server.db = {
# A collection of the known build targets # A collection of the known build targets
create_server.diff_targets = {} 'diff_targets': {},
# Build status panel data # Build status panel data
create_server.panel_data = {} 'panel_data': {},
# Whether the last status check of master was green # Whether the last status check of master was green
create_server.master_is_green = True 'master_is_green': True,
}
# If db_file_no_ext is not None, attempt to restore old database state
if db_file_no_ext:
db_file = db_file_no_ext + '.db'
app.logger.info("Loading persisted state file '{}'...".format(db_file))
if os.path.exists(db_file):
with shelve.open(db_file_no_ext, flag='r') as db:
for key in create_server.db.keys():
if key in db:
create_server.db[key] = db[key]
app.logger.info(
"Restored key '{}' from persisted state in file '{}'".format(
key, db_file))
else:
app.logger.info(
"Persisted state file '{}' does not exist. It will be created when written to.".format(db_file))
app.logger.info("Done")
else:
app.logger.warning(
"No database file specified. State will not be persisted.")
def saveDatabase():
FabienUnsubmitted
Not Done
Inline Actions

You should make it a decorator, so you won't have to call it from everywhere (untested):

def persist(func):
    def call_and_save_database(*args, **kwargs):
        # Call our decorated function
        func_ret = func(*args, **kwargs)

        # Save the database as needed
        if db_file_no_ext:
            with shelve.open(db_file_no_ext) as db:
                for key in create_server.db.keys():
                    db[key] = create_server.db[key]
                app.logger.debug("Persisted current state")
        else:
            app.logger.debug(
                "No database file specified. Persisting state is being skipped.")

        # Forward the return value
        return func_ret

    return call_and_save_database

@persist
def build_diff():

Note that I changed the logger level to debug, otherwise you can expect a ton of pollution to your logs.

Fabien: You should make it a decorator, so you won't have to call it from everywhere (untested)…
if db_file_no_ext:
with shelve.open(db_file_no_ext) as db:
for key in create_server.db.keys():
db[key] = create_server.db[key]
app.logger.info("Persisted current state")
else:
app.logger.warning(
"No database file specified. Persisting state is being skipped.")
# This decorator specifies an HMAC secret environment variable to use for verifying # This decorator specifies an HMAC secret environment variable to use for verifying
# requests for the given route. Currently, we're using Phabricator to trigger these # requests for the given route. Currently, we're using Phabricator to trigger these
# routes as webhooks, and a separate HMAC secret is required for each hook. # routes as webhooks, and a separate HMAC secret is required for each hook.
# Phabricator does not support basic auth for webhooks, so HMAC must be # Phabricator does not support basic auth for webhooks, so HMAC must be
# used instead. # used instead.
def verify_hmac(secret_env): def verify_hmac(secret_env):
def decorator(fn): def decorator(fn):
▲ Show 20 Lines • Show All 109 Lines • ▼ Show 20 Lines def build():
properties = None properties = None
if abcBuildName: if abcBuildName:
properties = [{ properties = [{
'name': 'env.ABC_BUILD_NAME', 'name': 'env.ABC_BUILD_NAME',
'value': abcBuildName, 'value': abcBuildName,
}] }]
build_id = tc.trigger_build(buildTypeId, ref, PHID, properties)['id'] build_id = tc.trigger_build(buildTypeId, ref, PHID, properties)['id']
if PHID in create_server.diff_targets: if PHID in create_server.db['diff_targets']:
build_target = create_server.diff_targets[PHID] build_target = create_server.db['diff_targets'][PHID]
else: else:
build_target = BuildTarget(PHID) build_target = BuildTarget(PHID)
build_target.queue_build(build_id, abcBuildName) build_target.queue_build(build_id, abcBuildName)
create_server.diff_targets[PHID] = build_target create_server.db['diff_targets'][PHID] = build_target
saveDatabase()
return SUCCESS, 200 return SUCCESS, 200
@app.route("/buildDiff", methods=['POST']) @app.route("/buildDiff", methods=['POST'])
def build_diff(): def build_diff():
def get_mandatory_argument(argument): def get_mandatory_argument(argument):
value = request.args.get(argument, None) value = request.args.get(argument, None)
if value is None: if value is None:
raise AssertionError( raise AssertionError(
Show All 15 Lines def build_diff():
builds = [ builds = [
k for k, k for k,
v in config.get( v in config.get(
'builds', 'builds',
{}).items() if v.get( {}).items() if v.get(
'runOnDiff', 'runOnDiff',
False)] False)]
if target_phid in create_server.diff_targets: if target_phid in create_server.db['diff_targets']:
build_target = create_server.diff_targets[target_phid] build_target = create_server.db['diff_targets'][target_phid]
else: else:
build_target = BuildTarget(target_phid) build_target = BuildTarget(target_phid)
for build_name in builds: for build_name in builds:
properties = [{ properties = [{
'name': 'env.ABC_BUILD_NAME', 'name': 'env.ABC_BUILD_NAME',
'value': build_name, 'value': build_name,
}] }]
build_id = tc.trigger_build( build_id = tc.trigger_build(
'BitcoinABC_BitcoinAbcStaging', 'BitcoinABC_BitcoinAbcStaging',
staging_ref, staging_ref,
target_phid, target_phid,
properties)['id'] properties)['id']
build_target.queue_build(build_id, build_name) build_target.queue_build(build_id, build_name)
create_server.diff_targets[target_phid] = build_target create_server.db['diff_targets'][target_phid] = build_target
saveDatabase()
return SUCCESS, 200 return SUCCESS, 200
@app.route("/land", methods=['POST']) @app.route("/land", methods=['POST'])
def land(): def land():
data = get_json_request_data(request) data = get_json_request_data(request)
revision = data['revision'] revision = data['revision']
if not revision: if not revision:
▲ Show 20 Lines • Show All 243 Lines • ▼ Show 20 Lines def update_build_status_panel(updated_build_type_id):
project_name_map = {} project_name_map = {}
for build in list(associated_builds.values()): for build in list(associated_builds.values()):
project_name_map[build['teamcity_project_id'] project_name_map[build['teamcity_project_id']
] = build['teamcity_project_name'] ] = build['teamcity_project_name']
# If the list of project names has changed (project was added, deleted # If the list of project names has changed (project was added, deleted
# or renamed, update the panel data accordingly. # or renamed, update the panel data accordingly.
(removed_projects, added_projects) = dict_xor( (removed_projects, added_projects) = dict_xor(
create_server.panel_data, project_ids, lambda key: {}) create_server.db['panel_data'], project_ids, lambda key: {})
saveDatabase()
# Log the project changes if any # Log the project changes if any
if (len(removed_projects) + len(added_projects)) > 0: if (len(removed_projects) + len(added_projects)) > 0:
app.logger.info( app.logger.info(
"Teamcity project list has changed.\nRemoved: {}\nAdded: {}".format( "Teamcity project list has changed.\nRemoved: {}\nAdded: {}".format(
removed_projects, removed_projects,
added_projects, added_projects,
) )
Show All 19 Lines def update_build_status_panel(updated_build_type_id):
build_status_message = build_info.get( build_status_message = build_info.get(
'statusText', 'statusText',
build_status.value) if build_status == BuildStatus.Failure else build_status.value build_status.value) if build_status == BuildStatus.Failure else build_status.value
return (build_status, build_status_message) return (build_status, build_status_message)
# Update the builds # Update the builds
for project_id, project_builds in sorted( for project_id, project_builds in sorted(
create_server.panel_data.items()): create_server.db['panel_data'].items()):
build_type_ids = [build['teamcity_build_type_id'] for build in list( build_type_ids = [build['teamcity_build_type_id'] for build in list(
associated_builds.values()) if build['teamcity_project_id'] == project_id] associated_builds.values()) if build['teamcity_project_id'] == project_id]
# If the list of builds has changed (build was added, deleted, # If the list of builds has changed (build was added, deleted,
# renamed, added to or removed from the items to display), update # renamed, added to or removed from the items to display), update
# the panel data accordingly. # the panel data accordingly.
(removed_builds, added_builds) = dict_xor( (removed_builds, added_builds) = dict_xor(
project_builds, project_builds,
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines def handle_build_result(buildName, buildTypeId, buildResult,
# The coverage report is not guaranteed to exist, in this # The coverage report is not guaranteed to exist, in this
# case teamcity will raise an exception. # case teamcity will raise an exception.
coverage_summary = None coverage_summary = None
if coverage_summary: if coverage_summary:
update_coverage_panel(coverage_summary) update_coverage_panel(coverage_summary)
# If we have a buildTargetPHID, report the status. # If we have a buildTargetPHID, report the status.
build_target = create_server.diff_targets.get(buildTargetPHID, None) build_target = create_server.db['diff_targets'].get(
buildTargetPHID, None)
if build_target is not None: if build_target is not None:
phab.update_build_target_status(build_target, buildId, status) phab.update_build_target_status(build_target, buildId, status)
send_harbormaster_build_link_if_required( send_harbormaster_build_link_if_required(
guest_url, guest_url,
build_target, build_target,
build_target.builds[buildId].name build_target.builds[buildId].name
) )
if build_target.is_finished(): if build_target.is_finished():
del create_server.diff_targets[buildTargetPHID] del create_server.db['diff_targets'][buildTargetPHID]
saveDatabase()
revisionPHID = phab.get_revisionPHID(branch) revisionPHID = phab.get_revisionPHID(branch)
buildInfo = tc.getBuildInfo(buildId) buildInfo = tc.getBuildInfo(buildId)
isAutomated = tc.checkBuildIsAutomated(buildInfo) isAutomated = tc.checkBuildIsAutomated(buildInfo)
if isAutomated and status == BuildStatus.Failure: if isAutomated and status == BuildStatus.Failure:
# Check if this failure is infrastructure-related # Check if this failure is infrastructure-related
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines def handle_build_result(buildName, buildTypeId, buildResult,
if status == BuildStatus.Success: if status == BuildStatus.Success:
updatedTask = phab.updateBrokenBuildTaskStatus( updatedTask = phab.updateBrokenBuildTaskStatus(
buildName, 'resolved') buildName, 'resolved')
if updatedTask: if updatedTask:
# Only message once all of master is green # Only message once all of master is green
(buildFailures, testFailures) = tc.getLatestBuildAndTestFailures( (buildFailures, testFailures) = tc.getLatestBuildAndTestFailures(
'BitcoinABC') 'BitcoinABC')
if len(buildFailures) == 0 and len(testFailures) == 0: if len(buildFailures) == 0 and len(testFailures) == 0:
if not create_server.master_is_green: if not create_server.db['master_is_green']:
create_server.master_is_green = True create_server.db['master_is_green'] = True
saveDatabase()
slackbot.postMessage( slackbot.postMessage(
'dev', "Master is green again.") 'dev', "Master is green again.")
if status == BuildStatus.Failure: if status == BuildStatus.Failure:
shortBuildUrl = tc.build_url( shortBuildUrl = tc.build_url(
"viewLog.html", "viewLog.html",
{ {
"buildId": buildId, "buildId": buildId,
Show All 24 Lines def handle_build_result(buildName, buildTypeId, buildResult,
if numRecentFailures >= 2: if numRecentFailures >= 2:
# This build may be flaky. Ping the channel with a # This build may be flaky. Ping the channel with a
# less-noisy message. # less-noisy message.
slackbot.postMessage('dev', slackbot.postMessage('dev',
"Build '{}' appears to be flaky: {}".format(buildName, shortBuildUrl)) "Build '{}' appears to be flaky: {}".format(buildName, shortBuildUrl))
return SUCCESS, 200 return SUCCESS, 200
# Only mark master as red for failures that are not flaky # Only mark master as red for failures that are not flaky
create_server.master_is_green = False create_server.db['master_is_green'] = False
saveDatabase()
commitHashes = buildInfo.getCommits() commitHashes = buildInfo.getCommits()
newTask = phab.createBrokenBuildTask( newTask = phab.createBrokenBuildTask(
buildName, guest_url, branch, commitHashes, 'rABC') buildName, guest_url, branch, commitHashes, 'rABC')
if newTask: if newTask:
# TODO: Add 'Reviewed by: <slack-resolved reviewer # TODO: Add 'Reviewed by: <slack-resolved reviewer
# names>' line # names>' line
▲ Show 20 Lines • Show All 74 Lines • Show Last 20 Lines