From 147caddf842e8a9382cb5f851aa10e1cf2b16c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 3 Dec 2024 21:40:13 +0200 Subject: [PATCH] Move master-docker-nonstandard to use common master setup --- master-docker-nonstandard/master.cfg | 206 ++++++++------------------- 1 file changed, 61 insertions(+), 145 deletions(-) diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 5219ecc8..7f6e8032 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -1,28 +1,39 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys from collections import defaultdict -sys.path.insert(0, "/srv/buildbot/master") -sys.setrecursionlimit(10000) - -from common_factories import * -from constants import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * +import docker +from buildbot.plugins import steps, util, worker +from buildbot.process.properties import Property +from common_factories import ( + addTests, + getHTMLLogString, + getQuickBuildFactory, + getRpmAutobakeFactory, + getSourceTarball, +) +from constants import MTR_ENV +from locks import getLocks +from master_common import base_master_config +from utils import ( + canStartBuild, + createVar, + createWorker, + dockerfile, + filterBranch, + hasAutobake, + hasBigtest, + hasEco, + hasFailed, + moveMTRLogs, + mtrJobsMultiplier, + nextBuild, + printEnv, + saveLogs, + savePackageIfBranchMatch, +) FQDN = os.environ["BUILDMASTER_WG_IP"] @@ -35,7 +46,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -43,61 +54,11 @@ config = {"private": {}} with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: exec(file.read(), config, {}) + # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - - -####### BUILDBOT SERVICES - -# 'services' is a list of BuildbotService items like reporter targets. The -# status of each build will be pushed to these targets. buildbot/reporters/*.py -# has a variety to choose from, like IRC bots. - - -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c["secretsProviders"] = [ - secrets.SecretInAFile( - dirname=os.environ["MASTER_CREDENTIALS_DIR"] - ) -] -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} +c = BuildmasterConfig = base_master_config(config) + mtrDbPool = util.EqConnectionPool( "MySQLdb", @@ -107,14 +68,6 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS - ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is @@ -149,8 +102,6 @@ def addWorker( # Docker workers -fqdn = os.environ["BUILDMASTER_WG_IP"] - ## hz-bbw2-docker c["workers"].append( worker.DockerLatentWorker( @@ -159,7 +110,7 @@ c["workers"].append( docker_host=config["private"]["docker_workers"]["hz-bbw2-docker"], dockerfile=open("dockerfiles/eco-php-ubuntu-2004.dockerfile").read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -182,7 +133,7 @@ c["workers"].append( "dockerfiles/eco-pymysql-python-3-9-slim-buster.dockerfile" ).read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -203,7 +154,7 @@ c["workers"].append( docker_host=config["private"]["docker_workers"]["hz-bbw2-docker"], dockerfile=open("dockerfiles/eco-mysqljs-nodejs15-buster.dockerfile").read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -224,12 +175,11 @@ c["workers"].append( "bm-bbw1-docker-ubuntu-2004", None, docker_host=config["private"]["docker_workers"]["bm-bbw1-docker"], - image=os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + image=os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", followStartupLogs=False, autopull=True, alwaysPull=True, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "20G", "ulimits": [ @@ -246,8 +196,7 @@ addWorker( "hz-bbw", 6, "bigtest-ubuntu-2004", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=20, save_packages=False, ) @@ -258,8 +207,7 @@ for w_name in ["ppc64le-osuosl-bbw"]: w_name, 1, "ubuntu-2004", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, save_packages=True, shm_size="20G", @@ -268,8 +216,7 @@ for w_name in ["ppc64le-osuosl-bbw"]: w_name, 1, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=30, save_packages=True, ) @@ -286,8 +233,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "aocc-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-aocc", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-aocc", jobs=jobs, save_packages=False, ) @@ -295,8 +241,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "asan-ubuntu-2404", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu24.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu24.04", jobs=jobs, save_packages=False, ) @@ -304,8 +249,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "icc-ubuntu-2204", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu22.04-icc", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04-icc", jobs=jobs, save_packages=False, ) @@ -313,8 +257,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "ubuntu-2004", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=jobs, save_packages=True, ) @@ -324,8 +267,7 @@ addWorker( "amd-bbw", 1, "valgrind-fedora-40", - os.environ["CONTAINER_REGISTRY_URL"] - + "fedora40-valgrind", + os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -333,8 +275,7 @@ addWorker( "amd-bbw", 2, "valgrind-fedora-40", - os.environ["CONTAINER_REGISTRY_URL"] - + "fedora40-valgrind", + os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -342,8 +283,7 @@ addWorker( "hz-bbw", 6, "valgrind-fedora-40", - os.environ["CONTAINER_REGISTRY_URL"] - + "fedora40-valgrind", + os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -352,8 +292,7 @@ addWorker( "hz-bbw", 1, "msan-clang-16-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-msan-clang-16", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=20, save_packages=False, ) @@ -361,8 +300,7 @@ addWorker( "hz-bbw", 4, "msan-clang-16-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-msan-clang-16", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=20, save_packages=False, ) @@ -370,8 +308,7 @@ addWorker( "hz-bbw", 5, "msan-clang-16-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-msan-clang-16", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=30, save_packages=False, ) @@ -380,8 +317,7 @@ addWorker( "hz-bbw", 2, "debian-12", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian12", + os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=20, save_packages=False, ) @@ -389,8 +325,7 @@ addWorker( "hz-bbw", 5, "debian-12", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian12", + os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=20, save_packages=False, ) @@ -399,8 +334,7 @@ addWorker( "aarch64-bbw", 6, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=10, save_packages=True, ) @@ -408,8 +342,7 @@ addWorker( "aarch64-bbw", 6, "debian-10-bintar", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian10-bintar", + os.environ["CONTAINER_REGISTRY_URL"] + "debian10-bintar", jobs=10, save_packages=True, ) @@ -418,8 +351,7 @@ addWorker( "hz-bbw", 5, "centos-7-bintar", - os.environ["CONTAINER_REGISTRY_URL"] - + "centos7-bintar", + os.environ["CONTAINER_REGISTRY_URL"] + "centos7-bintar", jobs=10, save_packages=True, ) @@ -428,8 +360,7 @@ addWorker( "s390x-bbw", 1, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -438,8 +369,7 @@ addWorker( "s390x-bbw", 2, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -448,8 +378,7 @@ addWorker( "s390x-bbw", 3, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -460,8 +389,7 @@ c["workers"].append( "release-prep-docker", None, docker_host=config["private"]["docker_workers"]["release-prep-docker"], - image=os.environ["CONTAINER_REGISTRY_URL"] - + "debian12-release", + image=os.environ["CONTAINER_REGISTRY_URL"] + "debian12-release", followStartupLogs=False, autopull=True, alwaysPull=True, @@ -1794,15 +1722,3 @@ c["builders"].append( factory=f_prep_local, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -}