diff --git a/master-galera/master.cfg b/master-galera/master.cfg index ab49aa23..65ef1f54 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -1,26 +1,18 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from twisted.internet import defer -import sys +# git branch filter using fnmatch +import fnmatch import os -import docker -import yaml from datetime import timedelta from collections import defaultdict -sys.setrecursionlimit(10000) +from buildbot.plugins import schedulers, steps, util, worker +from constants import ALL_PLATFORMS, BUILDERS_GALERA, OS_INFO +from master_common import base_master_config +from utils import createWorker, savePackageIfBranchMatch, waitIfStaging, nextBuild -sys.path.append(os.getcwd() + "/..") -from constants import * -from utils import * cfg_dir = os.path.abspath(os.path.dirname(__file__)) @@ -31,7 +23,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -41,41 +33,10 @@ with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - +c = BuildmasterConfig = base_master_config(port) FQDN = os.environ["BUILDMASTER_WG_IP"] -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} - -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None ####### SCHEDULERS @@ -84,9 +45,6 @@ c["buildbotNetUsageData"] = None BRANCHES_MAIN = ["mariadb-3.x", "mariadb-4.x", "bb-*"] SAVED_PACKAGE_BRANCHES_GALERA = ["mariadb-3.x", "mariadb-4.x", "bb-*"] -# git branch filter using fnmatch -import fnmatch - def upstream_branch_fn(branch): return ( @@ -98,9 +56,11 @@ def upstream_branch_fn(branch): ) - +# Override schedulers. +# TODO(cvicentiu): Move this to base_master_config maybe? c["schedulers"] = [] + schedulerTrigger = schedulers.AnyBranchScheduler( name="s_upstream_galera", change_filter=util.ChangeFilter( @@ -172,6 +132,7 @@ GALERA_PACKAGES = os.environ["GALERA_PACKAGES_DIR"] workers = defaultdict(list) + def addWorker( worker_name_prefix, worker_id, @@ -256,7 +217,7 @@ for platform in ALL_PLATFORMS: def dpkgDeb(): - return ShellCommand( + return steps.ShellCommand( name="apt-ftparchive", haltOnFailure=True, command=[ @@ -275,14 +236,14 @@ def dpkgDeb(): """ ), ], - doStepIf=(lambda step: - savePackageIfBranchMatch(step, - SAVED_PACKAGE_BRANCHES_GALERA)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA) + ), ) def rpmSave(): - return ShellCommand( + return steps.ShellCommand( name="move rpm files", haltOnFailure=True, command=[ @@ -292,13 +253,13 @@ def rpmSave(): """set -e mkdir -p rpms srpms cp `find *.rpm -maxdepth 1 -type f` rpms - find rpms -type f -exec sha256sum {} \; | sort > sha256sums.txt + find rpms -type f -exec sha256sum {} \\; | sort > sha256sums.txt """ ), ], - doStepIf=(lambda step: - savePackageIfBranchMatch(step, - SAVED_PACKAGE_BRANCHES_GALERA)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA) + ), ) @@ -380,9 +341,9 @@ EOF """, url=os.environ["ARTIFACTS_URL"], ), - doStepIf=(lambda step: - savePackageIfBranchMatch(step, - SAVED_PACKAGE_BRANCHES_GALERA)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA) + ), ) ) f_deb_build.addStep( @@ -439,9 +400,9 @@ EOF """, url=os.environ["ARTIFACTS_URL"], ), - doStepIf=(lambda step: - savePackageIfBranchMatch(step, - SAVED_PACKAGE_BRANCHES_GALERA)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA) + ), ) ) f_rpm_build.addStep( @@ -450,6 +411,7 @@ f_rpm_build.addStep( ) ) + ####### BUILDERS LIST c["builders"] = [] @@ -467,6 +429,7 @@ c["builders"].append( ) ) + for os_i in OS_INFO: if "install_only" in OS_INFO[os_i] and OS_INFO[os_i]["install_only"]: continue @@ -502,15 +465,3 @@ for os_i in OS_INFO: factory=factory, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -}