From 652d7d8b2cfbda2e4ea7aac26c693f677e55ecf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 3 Dec 2024 20:24:33 +0200 Subject: [PATCH] Reorganize master.cfg to simplify config specification --- master.cfg | 136 +++++++++++++++++++++++------------------------------ 1 file changed, 58 insertions(+), 78 deletions(-) diff --git a/master.cfg b/master.cfg index 30ec8eaa..ba0ba7ae 100644 --- a/master.cfg +++ b/master.cfg @@ -16,8 +16,8 @@ from common_factories import ( from constants import ( BUILDERS_GALERA_MTR, GITHUB_STATUS_BUILDERS, - SAVED_PACKAGE_BRANCHES, OS_INFO, + SAVED_PACKAGE_BRANCHES, ) from locks import getLocks from schedulers_definition import SCHEDULERS @@ -27,8 +27,8 @@ from utils import ( createWorker, dockerfile, hasDockerLibrary, - hasPackagesGenerated, hasInstall, + hasPackagesGenerated, hasUpgrade, ls2string, nextBuild, @@ -39,7 +39,7 @@ from utils import ( cfg_dir = os.path.abspath(os.path.dirname(__file__)) # Autogen master, see buildbot.tac for why this is the case. -base_dir = os.path.abspath(f'{cfg_dir}/../../') +base_dir = os.path.abspath(f"{cfg_dir}/../../") with open(os.path.join(cfg_dir, "master-config.yaml"), "r") as file: master_config = yaml.safe_load(file) @@ -52,60 +52,63 @@ with open(os.path.join(cfg_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - - -####### -# BUILDBOT SERVICES -####### - -# 'services' is a list of BuildbotService items like reporter targets. The -# status of each build will be pushed to these targets. buildbot/reporters/*.py -# has a variety to choose from, like IRC bots. - - -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c["secretsProviders"] = [ - secrets.SecretInAFile(dirname=os.environ["MASTER_CREDENTIALS_DIR"]) -] - -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -c["protocols"] = {"pb": {"port": os.environ["PORT"]}} - -####### DB URL - -c["db"] = { +c = BuildmasterConfig = { + ####### + # PROJECT IDENTITY + ####### + # the 'title' string will appear at the top of this buildbot installation's + "title": os.getenv("TITLE", default="MariaDB CI"), + # home pages (linked to the 'titleURL'). + "titleURL": os.getenv("TITLE_URL", default="https://github.com/MariaDB/server"), + # the 'buildbotURL' string should point to the location where the + # buildbot's internal web server is visible. This typically uses the port + # number set in the 'www' entry below, but with an externally-visible host + # name which the buildbot cannot figure out without some help. + "buildbotURL": os.getenv( + "BUILDMASTER_URL", default="https://buildbot.mariadb.org/" + ), + # 'services' is a list of BuildbotService items like reporter targets. The + # status of each build will be pushed to these targets. buildbot/reporters/*.py + # has a variety to choose from, like IRC bots. + "services": [ + reporters.GitHubStatusPush( + token=config["private"]["gh_mdbci"]["access_token"], + context=util.Interpolate("buildbot/%(prop:buildername)s"), + startDescription="Build started.", + endDescription="Build done.", + verbose=True, + builders=GITHUB_STATUS_BUILDERS, + ) + ], + "secretsProviders": [ + secrets.SecretInAFile(dirname=os.environ["MASTER_CREDENTIALS_DIR"]) + ], + # 'protocols' contains information about protocols which master will use + # for communicating with workers. You must define at least 'port' option + # that workers could connect to your master with this protocol. + # 'port' must match the value configured into the workers (with their + # --master option) + "protocols": {"pb": {"port": os.environ["PORT"]}}, # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] + "db": {"db_url": config["private"]["db_url"]}, + # Disable net usage reports from being sent to buildbot.net + "buildbotNetUsageData": None, + # Configure the Schedulers, which decide how to react to incoming changes. + "schedulers": SCHEDULERS, + "logEncoding": "utf-8", + "multiMaster": True, + "mq": { + # Need to enable multimaster aware mq. Wamp is the only option for now. + "type": "wamp", + "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), + "realm": "realm1", + # valid are: none, critical, error, warn, info, debug, trace + "wamp_debug_level": "info", + }, } +####### DB URL + mtrDbPool = util.EqConnectionPool( "MySQLdb", config["private"]["db_host"], @@ -114,14 +117,6 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS - ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is @@ -150,10 +145,7 @@ for w_name in master_config["workers"]: if image_tag.startswith("ubuntu"): image_tag = image_tag[:-2] + "." + image_tag[-2:] - quay_name = ( - os.environ["CONTAINER_REGISTRY_URL"] - + image_tag - ) + quay_name = os.environ["CONTAINER_REGISTRY_URL"] + image_tag if builder.startswith("x86"): os_name += "-i386" quay_name += "-386" @@ -373,15 +365,3 @@ for builder in master_config["builders"]: factory=factory_instance, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -}