From 99c2c37aef9462db025bd27be7ce66b6eb17b620 Mon Sep 17 00:00:00 2001 From: Vlad Bogolin Date: Thu, 10 Aug 2023 23:34:33 +0300 Subject: [PATCH] Use env vars to select between dev and prod --- .gitignore | 1 + Dockerfile | 2 +- constants.py | 10 ++-- docker-compose/dev.env | 4 ++ docker-compose/docker-compose.yaml | 50 +++++++++++++++++ docker-compose/generate-config.py | 81 ++++++++++++++++++++-------- docker-compose/prod.env | 4 ++ master-docker-nonstandard/master.cfg | 15 +++--- master-galera/dockerfiles | 1 - master-galera/master.cfg | 35 ++++++------ master-libvirt/master.cfg | 15 +++--- master-nonlatent/master.cfg | 18 +++---- master-protected-branches/master.cfg | 18 +++---- master-web/master.cfg | 13 ++--- master.cfg | 18 +++---- rsync.exclude | 1 + 16 files changed, 188 insertions(+), 98 deletions(-) create mode 100644 docker-compose/dev.env create mode 100644 docker-compose/prod.env delete mode 120000 master-galera/dockerfiles diff --git a/.gitignore b/.gitignore index 9e86ac1d..c35c6698 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ venv workers autogen master-config.yaml +.venv diff --git a/Dockerfile b/Dockerfile index 3a66575a..cae924b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ && if [ "$master_type" = "master-web" ]; then \ make frontend; \ fi \ - && pip install --no-cache-dir wheel \ + && pip install --no-cache-dir python-dotenv wheel \ && pip install --no-cache-dir autobahn==20.7.1 PyJWT==1.7.1 \ && cd master && python setup.py bdist_wheel \ && pip install --no-cache-dir dist/*.whl \ diff --git a/constants.py b/constants.py index c5c7b9c2..a9948507 100644 --- a/constants.py +++ b/constants.py @@ -170,15 +170,15 @@ builders_upgrade = [] builders_autobake = [] all_platforms = set() -for os in os_info: - for arch in os_info[os]['arch']: +for os_i in os_info: + for arch in os_info[os_i]['arch']: all_platforms.add(arch) - builder_name_autobake = arch + '-' + os + '-' + os_info[os]['type'] + '-autobake' + builder_name_autobake = arch + '-' + os_i + '-' + os_info[os_i]['type'] + '-autobake' builders_autobake.append(builder_name_autobake) # Currently there are no VMs for x86 and s390x and OpenSUSE and SLES addInstall = True - if 'has_install' in os_info[os]: - addInstall = os_info[os]['has_install'] + if 'has_install' in os_info[os_i]: + addInstall = os_info[os_i]['has_install'] if arch not in ['s390x', 'x86'] and addInstall: builders_install.append(builder_name_autobake + '-install') builders_upgrade.append(builder_name_autobake + '-minor-upgrade') diff --git a/docker-compose/dev.env b/docker-compose/dev.env new file mode 100644 index 00000000..be21d369 --- /dev/null +++ b/docker-compose/dev.env @@ -0,0 +1,4 @@ +TITLE="MariaDB CI" +TITLE_URL=https://github.com/MariaDB/server +BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ +MQ_ROUTER_URL=ws://crossbar:8085/ws \ No newline at end of file diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 77c7634b..48e40afb 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -31,6 +31,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master-web restart: unless-stopped container_name: master-web + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -49,6 +54,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_aarch64-master-0 + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -69,6 +79,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_amd64-master-0 + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -89,6 +104,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_amd64-master-1 + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -109,6 +129,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_ppc64le-master-0 + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -129,6 +154,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_s390x-master-0 + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -149,6 +179,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_x86-master-0 + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -169,6 +204,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: master-docker-nonstandard + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -189,6 +229,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: master-galera + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -209,6 +254,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: master-protected-branches + environment: + - TITLE=MariaDB CI + - TITLE_URL=https://github.com/MariaDB/server + - BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ + - MQ_ROUTER_URL=ws://crossbar:8085/ws volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py index f77f21ed..ceddff92 100755 --- a/docker-compose/generate-config.py +++ b/docker-compose/generate-config.py @@ -1,6 +1,10 @@ #!/usr/bin/env python3 +import argparse +import os -master_directories = [ +from dotenv import load_dotenv + +MASTER_DIRECTORIES = [ "autogen/aarch64-master-0", "autogen/amd64-master-0", "autogen/amd64-master-1", @@ -15,7 +19,7 @@ "master-protected-branches", ] -start_template = """ +START_TEMPLATE = """ --- version: "3.7" services: @@ -63,7 +67,7 @@ - crossbar """ -docker_compose_template = """ +DOCKER_COMPOSE_TEMPLATE = """ {master_name}: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped @@ -85,7 +89,7 @@ - crossbar """ -end_template = """ +END_TEMPLATE = """ networks: net_front: driver: bridge @@ -108,21 +112,54 @@ com.docker.network.bridge.name: "br_bb_back" """ -# Generate startup scripts and Docker Compose pieces for each master directory -with open("docker-compose.yaml", "w") as f: - f.write("# This is an autogenerated file. Do not edit it manually. Use `python generate-config.py` instead.") - f.write(start_template) - port = 8011 - for master_directory in master_directories: - master_name = master_directory.replace("/", "_") - - # Generate Docker Compose piece - docker_compose_piece = docker_compose_template.format( - master_name=master_name, - master_directory=master_directory, - port=port, - ) - port += 1 - - f.write(docker_compose_piece) - f.write(end_template) +# Function to construct environment section for Docker Compose +def construct_env_section(env_vars): + env_section = " environment:\n" + for key, value in env_vars.items(): + env_section += f" - {key}={value}\n" + return env_section.rstrip('\n') + +def main(args): + # Load environment variables from the corresponding .env file + env_file = "prod.env" if args.env == "prod" else "dev.env" + load_dotenv(env_file) + + env_vars = { + "TITLE": os.getenv("TITLE"), + "TITLE_URL": os.getenv("TITLE_URL"), + "BUILDMASTER_URL": os.getenv("BUILDMASTER_URL"), + "MQ_ROUTER_URL": os.getenv("MQ_ROUTER_URL") + } + + # Modify the start_template to include the environment variables for master-web + master_env_section = construct_env_section(env_vars) + start_template = START_TEMPLATE.replace("container_name: master-web", f"container_name: master-web\n{master_env_section}") + + # Modify the docker_compose_template to include the environment variables + docker_compose_template = DOCKER_COMPOSE_TEMPLATE.replace("container_name: {master_name}", f"container_name: {{master_name}}\n{master_env_section}") + + # Generate startup scripts and Docker Compose pieces for each master directory + with open("docker-compose.yaml", "w") as f: + f.write("# This is an autogenerated file. Do not edit it manually. Use `python generate-config.py` instead.") + f.write(start_template) + port = 8011 + for master_directory in MASTER_DIRECTORIES: + master_name = master_directory.replace("/", "_") + + # Generate Docker Compose piece + docker_compose_piece = docker_compose_template.format( + master_name=master_name, + master_directory=master_directory, + port=port, + ) + port += 1 + + f.write(docker_compose_piece) + f.write(END_TEMPLATE) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate Docker Compose configuration.") + parser.add_argument("--env", choices=["prod", "dev"], default="dev", help="Choose the environment (prod/dev). Default is dev.") + + args = parser.parse_args() + main(args) diff --git a/docker-compose/prod.env b/docker-compose/prod.env new file mode 100644 index 00000000..dbbad0a7 --- /dev/null +++ b/docker-compose/prod.env @@ -0,0 +1,4 @@ +TITLE="MariaDB CI" +TITLE_URL=https://github.com/MariaDB/server +BUILDMASTER_URL=https://buildbot.mariadb.org/ +MQ_ROUTER_URL=ws://localhost:8085/ws \ No newline at end of file diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 2f3bd721..4edf7207 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.path.insert(0, '/srv/buildbot/master') sys.setrecursionlimit(10000) @@ -51,15 +53,14 @@ c['services'].append(gs) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -758,7 +759,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-galera/dockerfiles b/master-galera/dockerfiles deleted file mode 120000 index f8f80794..00000000 --- a/master-galera/dockerfiles +++ /dev/null @@ -1 +0,0 @@ -../dockerfiles \ No newline at end of file diff --git a/master-galera/master.cfg b/master-galera/master.cfg index 8867b5b2..7ca14831 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -7,11 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys -import os + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -32,18 +33,14 @@ exec(open("../master-private.cfg").read(), config, { }) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.dev.mariadb.org/" - -# Custom plugin -# exec(open("grid.py").read()) +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -214,19 +211,19 @@ f_rpm_build.addStep(steps.ShellCommand(name="cleanup", command="rm -r * .* 2> /d ####### BUILDERS LIST c['builders'] = [] -for os in os_info: - for arch in os_info[os]['arch']: - builder_name = 'gal-' + arch + '-' + os +for os_i in os_info: + for arch in os_info[os_i]['arch']: + builder_name = 'gal-' + arch + '-' + os_i assert builder_name in builders_galera - worker_name = arch + '-bbw-docker-' + os + worker_name = arch + '-bbw-docker-' + os_i if arch == 'amd64': - worker_name = 'x64-bbw-docker-' + os + worker_name = 'x64-bbw-docker-' + os_i if arch == 'x86': - worker_name = 'x64-bbw-docker-' + os + '-i386' + worker_name = 'x64-bbw-docker-' + os_i + '-i386' - if os_info[os]['type'] == 'rpm': + if os_info[os_i]['type'] == 'rpm': factory = f_rpm_build else: factory = f_deb_build @@ -234,7 +231,7 @@ for os in os_info: c['builders'].append( util.BuilderConfig(name=builder_name, workernames=workers[worker_name], - tags=[os, "galera", "gcc"], + tags=[os_i, "galera", "gcc"], collapseRequests=True, nextBuild=nextBuild, factory=factory)) @@ -245,7 +242,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://crossbar:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 7279414f..b47ffb4c 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -31,15 +33,14 @@ exec(open("../master-private.cfg").read(), config, { }) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.dev.mariadb.org/" +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -287,7 +288,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://crossbar:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index dd2c82c9..d9816dec 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -36,18 +38,14 @@ exec(open("../master-private.cfg").read(), config, { }) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" - -# Custom plugin -# exec(open("grid.py").read()) +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -344,7 +342,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://crossbar:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 26934121..ec6cda3b 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -52,18 +54,14 @@ c['services'].append(gs) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" - -# Custom plugin -# exec(open("grid.py").read()) +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -339,7 +337,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-web/master.cfg b/master-web/master.cfg index bcc8ff6e..8d173c59 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -7,11 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer + +import docker import os import sys -import docker -from datetime import timedelta sys.setrecursionlimit(10000) @@ -27,9 +28,9 @@ exec(open("../master-private.cfg").read(), config, { }) ####### PROJECT IDENTITY -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" -c['buildbotURL'] = "https://buildbot.dev.mariadb.org/" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # minimalistic config to activate web UI c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={}), custom_templates_dir='templates') @@ -78,7 +79,7 @@ c['multiMaster'] = True # Need to enable multimaster aware mq. Wamp is the only option for now. c['mq'] = { 'type' : 'wamp', - 'router_url': 'ws://crossbar:8080/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master.cfg b/master.cfg index 75895872..3b122891 100644 --- a/master.cfg +++ b/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -55,18 +57,14 @@ c['services'].append(gs) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" - -# Custom plugin -# exec(open("grid.py").read()) +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -262,7 +260,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://crossbar:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/rsync.exclude b/rsync.exclude index 09ad3956..645d78b5 100644 --- a/rsync.exclude +++ b/rsync.exclude @@ -6,3 +6,4 @@ ci_build_images docker-compose/logs docker-compose/mariadb master-private.cfg +dockerfiles \ No newline at end of file