Use env vars to select between dev and prod

.github/workflows/deploy.yml

@@ -47,15 +47,15 @@ jobs:
           github.repository == 'MariaDB/buildbot'
         run: |
           rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.SERVER_PORT }}" ./ ${{ secrets.USER }}@${{ secrets.SERVER_IP }}:/srv/dev/
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py"
+          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py --env=dev"
           ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/ && ./define_masters.py"
           ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ln -sf .. buildbot"
       - name: start stack
         if: >
           github.ref == 'refs/heads/dev' &&
           github.repository == 'MariaDB/buildbot'
         run: |
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose up -d"
+          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose --env-file .env.dev up -d"
       - name: clean
         if: >
           always() &&

.gitignore

@@ -8,3 +8,4 @@ venv
 workers
 autogen
 master-config.yaml
+.venv

Dockerfile

@@ -35,7 +35,7 @@ RUN apt-get update \
     && if [ "$master_type" = "master-web" ]; then \
       make frontend; \
     fi \
-    && pip install --no-cache-dir wheel \
+    && pip install --no-cache-dir python-dotenv wheel \
     && pip install --no-cache-dir autobahn==20.7.1 PyJWT==1.7.1 \
     && cd master && python setup.py bdist_wheel \
     && pip install --no-cache-dir dist/*.whl \

constants.py

@@ -170,15 +170,15 @@
 builders_upgrade = []
 builders_autobake = []
 all_platforms = set()
-for os in os_info:
-    for arch in os_info[os]['arch']:
+for os_i in os_info:
+    for arch in os_info[os_i]['arch']:
         all_platforms.add(arch)
-        builder_name_autobake = arch + '-' + os + '-' + os_info[os]['type'] + '-autobake'
+        builder_name_autobake = arch + '-' + os_i + '-' + os_info[os_i]['type'] + '-autobake'
         builders_autobake.append(builder_name_autobake)
         # Currently there are no VMs for x86 and s390x and OpenSUSE and SLES
         addInstall = True
-        if 'has_install' in os_info[os]:
-            addInstall = os_info[os]['has_install']
+        if 'has_install' in os_info[os_i]:
+            addInstall = os_info[os_i]['has_install']
         if arch not in ['s390x', 'x86'] and addInstall:
             builders_install.append(builder_name_autobake + '-install')
             builders_upgrade.append(builder_name_autobake + '-minor-upgrade')

define_masters.py

@@ -46,7 +46,12 @@
             yaml.dump(master_config, f)
 
         shutil.copyfile('master.cfg', dir_path + '/master.cfg')
-        shutil.copyfile('master-private.cfg', dir_path + '/master-private.cfg')
+        symlink_path = dir_path + '/master-private.cfg'
+
+        if not os.path.exists(symlink_path):
+            os.symlink('master-private.cfg', symlink_path)
+        else:
+            print(f"Symlink {symlink_path} already exists. Skipping...")
 
         buildbot_tac = open("buildbot.tac", "r").read() % master_config['log_name']
         with open(dir_path + '/buildbot.tac', 'w') as f:

docker-compose/.env

@@ -0,0 +1,4 @@
+TITLE="MariaDB CI"
+TITLE_URL=https://github.com/MariaDB/server
+BUILDMASTER_URL=https://buildbot.mariadb.org/
+MQ_ROUTER_URL=ws://localhost:8085/ws

docker-compose/.env.dev

@@ -0,0 +1,4 @@
+TITLE="MariaDB CI"
+TITLE_URL=https://github.com/MariaDB/server
+BUILDMASTER_URL=https://buildbot.dev.mariadb.org/
+MQ_ROUTER_URL=ws://crossbar:8085/ws

docker-compose/docker-compose.yaml

@@ -31,6 +31,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master-web
     restart: unless-stopped
     container_name: master-web
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -49,6 +54,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: autogen_aarch64-master-0
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -69,6 +79,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: autogen_amd64-master-0
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -89,6 +104,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: autogen_amd64-master-1
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -109,6 +129,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: autogen_ppc64le-master-0
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -129,6 +154,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: autogen_s390x-master-0
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -149,6 +179,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: autogen_x86-master-0
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -169,6 +204,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: master-docker-nonstandard
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -189,6 +229,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: master-galera
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master
@@ -209,6 +254,11 @@ services:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
     container_name: master-protected-branches
+    environment:
+      - MQ_ROUTER_URL
+      - TITLE_URL
+      - TITLE
+      - BUILDMASTER_URL
     volumes:
       - ./logs:/var/log/buildbot
       - ./buildbot/:/srv/buildbot/master

docker-compose/generate-config.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
+import argparse
+import os
 
-master_directories = [
+from dotenv import load_dotenv
+
+MASTER_DIRECTORIES = [
     "autogen/aarch64-master-0",
     "autogen/amd64-master-0",
     "autogen/amd64-master-1",
@@ -15,7 +19,7 @@
     "master-protected-branches",
 ]
 
-start_template = """
+START_TEMPLATE = """
 ---
 version: "3.7"
 services:
@@ -63,7 +67,7 @@
       - crossbar
 """
 
-docker_compose_template = """
+DOCKER_COMPOSE_TEMPLATE = """
   {master_name}:
     image: quay.io/mariadb-foundation/bb-master:master
     restart: unless-stopped
@@ -85,7 +89,7 @@
       - crossbar
 """
 
-end_template = """
+END_TEMPLATE = """
 networks:
   net_front:
     driver: bridge
@@ -108,21 +112,56 @@
       com.docker.network.bridge.name: "br_bb_back"
 """
 
-# Generate startup scripts and Docker Compose pieces for each master directory
-with open("docker-compose.yaml", "w") as f:
-    f.write("# This is an autogenerated file. Do not edit it manually. Use `python generate-config.py` instead.")
-    f.write(start_template)
-    port = 8011
-    for master_directory in master_directories:
-        master_name = master_directory.replace("/", "_")
-
-        # Generate Docker Compose piece
-        docker_compose_piece = docker_compose_template.format(
-            master_name=master_name,
-            master_directory=master_directory,
-            port=port,
-        )
-        port += 1
-
-        f.write(docker_compose_piece)
-    f.write(end_template)
+# Function to construct environment section for Docker Compose
+def construct_env_section(env_vars):
+    env_section = "    environment:\n"
+    for key, value in env_vars.items():
+        env_section += f"      - {key}\n"
+    return env_section.rstrip('\n')
+
+def main(args):
+    # Capture the current environment variables' keys
+    current_env_keys = set(os.environ.keys())
+
+    # Load environment variables from the corresponding .env file
+    env_file = ".env" if args.env == "prod" else ".env.dev"
+    load_dotenv(env_file)
+
+    # Determine the keys that were added by the .env file
+    new_keys = set(os.environ.keys()) - current_env_keys
+
+    # Extract only the variables from the .env file
+    env_vars = {key: os.getenv(key) for key in new_keys}
+
+    # Modify the start_template to include the environment variables for master-web
+    master_env_section = construct_env_section(env_vars)
+    start_template = START_TEMPLATE.replace("container_name: master-web", f"container_name: master-web\n{master_env_section}")
+
+    # Modify the docker_compose_template to include the environment variables
+    docker_compose_template = DOCKER_COMPOSE_TEMPLATE.replace("container_name: {master_name}", f"container_name: {{master_name}}\n{master_env_section}")
+
+    # Generate startup scripts and Docker Compose pieces for each master directory
+    with open("docker-compose.yaml", "w") as f:
+        f.write("# This is an autogenerated file. Do not edit it manually. Use `python generate-config.py` instead.")
+        f.write(start_template)
+        port = 8011
+        for master_directory in MASTER_DIRECTORIES:
+            master_name = master_directory.replace("/", "_")
+
+            # Generate Docker Compose piece
+            docker_compose_piece = docker_compose_template.format(
+                master_name=master_name,
+                master_directory=master_directory,
+                port=port,
+            )
+            port += 1
+
+            f.write(docker_compose_piece)
+        f.write(END_TEMPLATE)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate Docker Compose configuration.")
+    parser.add_argument("--env", choices=["prod", "dev"], default="dev", help="Choose the environment (prod/dev). Default is dev.")
+
+    args = parser.parse_args()
+    main(args)

master-docker-nonstandard/master.cfg

@@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom
 from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
 from buildbot.steps.source.github import GitHub
 from buildbot.process.remotecommand import RemoteCommand
+from datetime import timedelta
 from twisted.internet import defer
-import sys
+
 import docker
-from datetime import timedelta
+import os
+import sys
 
 sys.path.insert(0, '/srv/buildbot/master')
 sys.setrecursionlimit(10000)
@@ -51,15 +53,14 @@ c['services'].append(gs)
 
 # the 'title' string will appear at the top of this buildbot installation's
 # home pages (linked to the 'titleURL').
-c['title'] = "MariaDB CI"
-c['titleURL'] = "https://github.com/MariaDB/server"
+c['title'] = os.getenv('TITLE', default="MariaDB CI")
+c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server")
 
 # the 'buildbotURL' string should point to the location where the buildbot's
 # internal web server is visible. This typically uses the port number set in
 # the 'www' entry below, but with an externally-visible host name which the
 # buildbot cannot figure out without some help.
-
-c['buildbotURL'] = "https://buildbot.mariadb.org/"
+c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/")
 
 # 'protocols' contains information about protocols which master will use for
 # communicating with workers. You must define at least 'port' option that workers
@@ -758,7 +759,7 @@ c['multiMaster'] = True
 
 c['mq'] = {  # Need to enable multimaster aware mq. Wamp is the only option for now.
     'type' : 'wamp',
-    'router_url': 'ws://localhost:8085/ws',
+    'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'),
     'realm': 'realm1',
     # valid are: none, critical, error, warn, info, debug, trace
     'wamp_debug_level' : 'info'