Fix configuration issues upgrading Debian installs (#1636)

boefjes/debian/install

@@ -1,3 +1,2 @@
 packaging/deb/data/etc/kat/* etc/kat
-packaging/deb/data/usr/lib/systemd/system/* usr/lib/systemd/system
-packaging/deb/data/usr/bin/* usr/bin
+packaging/deb/data/usr/* usr

boefjes/packaging/deb/data/etc/kat/boefjes.conf

@@ -1,13 +1,12 @@
 QUEUE_URI=
 
-OCTOPOES_API=http://localhost:8001
-
-BYTES_API=http://localhost:8002
+# OCTOPOES_API=http://localhost:8001
+# BYTES_API=http://localhost:8002
 BYTES_USERNAME=bytes
 BYTES_PASSWORD=
 
 KATALOGUS_DB_URI=
-KATALOGUS_API=http://localhost:8003
-SCHEDULER_API=http://localhost:8004
+# KATALOGUS_API=http://localhost:8003
+# SCHEDULER_API=http://localhost:8004
 
-BOEFJES_LOG_CFG=/etc/kat/boefjes.logging.json
+# BOEFJES_LOG_CFG=/etc/kat/boefjes.logging.json

boefjes/packaging/deb/data/usr/bin/update-katalogus-db

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -ae
+source /usr/lib/kat/boefjes.defaults
 source /etc/kat/boefjes.conf
 cd /opt/venvs/kat-boefjes/lib/python*/site-packages
 

boefjes/packaging/deb/data/usr/lib/kat/boefjes.defaults

@@ -0,0 +1,6 @@
+OCTOPOES_API=http://localhost:8001
+BYTES_API=http://localhost:8002
+KATALOGUS_API=http://localhost:8003
+SCHEDULER_API=http://localhost:8004
+
+BOEFJES_LOG_CFG=/etc/kat/boefjes.logging.json

boefjes/packaging/deb/data/usr/lib/systemd/system/kat-boefjes.service

@@ -7,7 +7,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-boefjes
 WorkingDirectory=/opt/venvs/kat-boefjes/lib/python3.9/site-packages
-# TODO: don't load env here
+EnvironmentFile=/usr/lib/kat/boefjes.defaults
 EnvironmentFile=/etc/kat/boefjes.conf
 ExecStart=/opt/venvs/kat-boefjes/bin/python -m boefjes boefje
 Restart=on-failure

boefjes/packaging/deb/data/usr/lib/systemd/system/kat-katalogus.service

@@ -7,7 +7,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-katalogus
 WorkingDirectory=/opt/venvs/kat-boefjes/lib/python3.9/site-packages
-# TODO: don't load env here
+EnvironmentFile=/usr/lib/kat/boefjes.defaults
 EnvironmentFile=/etc/kat/boefjes.conf
 ExecStart=/opt/venvs/kat-boefjes/bin/python -m gunicorn \
           --access-logfile - \

boefjes/packaging/deb/data/usr/lib/systemd/system/kat-normalizers.service

@@ -7,7 +7,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-normalizers
 WorkingDirectory=/opt/venvs/kat-boefjes/lib/python3.9/site-packages
-# TODO: don't load env here
+EnvironmentFile=/usr/lib/kat/boefjes.defaults
 EnvironmentFile=/etc/kat/boefjes.conf
 ExecStart=/opt/venvs/kat-boefjes/bin/python -m boefjes normalizer
 Restart=on-failure

bytes/bytes/config.py

@@ -46,6 +46,10 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]:
                 logging.warning("Deprecation: %s is deprecated, use %s instead", old_name.upper(), new_name.upper())
                 d[new_name[len(env_prefix) :]] = env_vars[old_name]
 
+        # We previously accepted an empty value for this field
+        if "rfc3161_provider" in d and not d["rfc3161_provider"]:
+            del d["rfc3161_provider"]
+
         return d
 
 

bytes/debian/install

@@ -1,3 +1,2 @@
 packaging/deb/data/etc/kat/* etc/kat
-packaging/deb/data/usr/lib/systemd/system/* usr/lib/systemd/system
-packaging/deb/data/usr/bin/* usr/bin
+packaging/deb/data/usr/* usr

bytes/packaging/deb/data/etc/kat/bytes.conf

@@ -2,15 +2,14 @@
 BYTES_SECRET=
 BYTES_USERNAME=bytes
 BYTES_PASSWORD=
-BYTES_ACCESS_TOKEN_EXPIRE_MINUTES=1000
+# BYTES_ACCESS_TOKEN_EXPIRE_MINUTES=1000
 
 # Bytes DB
 BYTES_DB_URI=
 
 # QUEUE for messages other services in KAT listen to
 QUEUE_URI=
 
-# Optional environment variables
-BYTES_LOG_FILE=/var/log/kat-bytes/bytes.log
-BYTES_DATA_DIR=/var/lib/kat-bytes/
-BYTES_LOG_CFG=/etc/kat/bytes.logging.conf
+# BYTES_LOG_FILE=/var/log/kat-bytes/bytes.log
+# BYTES_DATA_DIR=/var/lib/kat-bytes/
+# BYTES_LOG_CFG=/etc/kat/bytes.logging.conf

bytes/packaging/deb/data/usr/bin/update-bytes-db

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -ae
+source /usr/lib/kat/bytes.defaults
 source /etc/kat/bytes.conf
 cd /opt/venvs/kat-bytes/lib/python*/site-packages
 /opt/venvs/kat-bytes/bin/python -m alembic --config bytes/alembic.ini upgrade head

bytes/packaging/deb/data/usr/lib/kat/bytes.defaults

@@ -0,0 +1,4 @@
+BYTES_ACCESS_TOKEN_EXPIRE_MINUTES=1000
+BYTES_LOG_FILE=/var/log/kat-bytes/bytes.log
+BYTES_DATA_DIR=/var/lib/kat-bytes/
+BYTES_LOG_CFG=/etc/kat/bytes.logging.conf

bytes/packaging/deb/data/usr/lib/systemd/system/kat-bytes.service

@@ -7,6 +7,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-bytes
 WorkingDirectory=/opt/venvs/kat-bytes/
+EnvironmentFile=/usr/lib/kat/bytes.defaults
 EnvironmentFile=/etc/kat/bytes.conf
 ExecStart=/opt/venvs/kat-bytes/bin/python -m gunicorn \
           --access-logfile - \

keiko/debian/install

@@ -1,5 +1,5 @@
 packaging/deb/data/etc/kat/* etc/kat
-packaging/deb/data/usr/lib/systemd/system/* usr/lib/systemd/system
+packaging/deb/data/usr/* usr
 templates/* /etc/kat/keiko/templates
 glossaries/* /etc/kat/keiko/glossaries
 assets/* /etc/kat/keiko/assets

keiko/packaging/deb/data/etc/kat/keiko.conf

@@ -1,14 +1,14 @@
-# File path to the log configuration file, default: "logging.json"
-KEIKO_LOG_CFG=/etc/kat/keiko.logging.json
+# File path to the log configuration file
+# KEIKO_LOG_CFG=/etc/kat/keiko.logging.json
 
-# File path to templates folder, default: "templates"
-KEIKO_TEMPLATES_FOLDER=/etc/kat/keiko/templates
+# File path to templates folder
+# KEIKO_TEMPLATES_FOLDER=/etc/kat/keiko/templates
 
-# File path to glossaries folder, default: "glossaries"
-KEIKO_GLOSSARIES_FOLDER=/etc/kat/keiko/glossaries
+# File path to glossaries folder
+# KEIKO_GLOSSARIES_FOLDER=/etc/kat/keiko/glossaries
 
-# File path to reports folder, default: "reports"
-KEIKO_REPORTS_FOLDER=/var/lib/kat-keiko/reports
+# File path to reports folder
+# KEIKO_REPORTS_FOLDER=/var/lib/kat-keiko/reports
 
-# File path to assets folder, default: "assets"
-KEIKO_ASSETS_FOLDER=/etc/kat/keiko/assets
+# File path to assets folder
+# KEIKO_ASSETS_FOLDER=/etc/kat/keiko/assets

keiko/packaging/deb/data/usr/lib/kat/keiko.defaults

@@ -0,0 +1,5 @@
+KEIKO_ASSETS_FOLDER=/etc/kat/keiko/assets
+KEIKO_GLOSSARIES_FOLDER=/etc/kat/keiko/glossaries
+KEIKO_LOG_CFG=/etc/kat/keiko.logging.json
+KEIKO_REPORTS_FOLDER=/var/lib/kat-keiko/reports
+KEIKO_TEMPLATES_FOLDER=/etc/kat/keiko/templates

keiko/packaging/deb/data/usr/lib/systemd/system/kat-keiko.service

@@ -7,6 +7,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-keiko
 WorkingDirectory=/opt/venvs/kat-keiko/
+EnvironmentFile=/usr/lib/kat/keiko.defaults
 EnvironmentFile=/etc/kat/keiko.conf
 ExecStart=/opt/venvs/kat-keiko/bin/python -m gunicorn \
           --access-logfile - \

mula/debian/install

@@ -1,3 +1,2 @@
 packaging/deb/data/etc/kat/* etc/kat
-packaging/deb/data/usr/lib/systemd/system/* usr/lib/systemd/system
-packaging/deb/data/usr/bin/* usr/bin
+packaging/deb/data/usr/* usr

mula/packaging/deb/data/etc/kat/mula.conf

@@ -1,19 +1,19 @@
 # Host address, default: localhost
-SCHEDULER_API_HOST=localhost
+# SCHEDULER_API_HOST=localhost
 
 # Host api server port, default: 8000
-SCHEDULER_API_PORT=8004
+# SCHEDULER_API_PORT=8004
 
 # File path to the log configuration file, default is "../../../logging.json"
-SCHEDULER_LOG_CFG=/etc/kat/mula.logging.json
+# SCHEDULER_LOG_CFG=/etc/kat/mula.logging.json
 
 # RabbitMQ host address
 QUEUE_URI=
 
 # Host url's of external service connectors
-KATALOGUS_API=http://localhost:8003
-BYTES_API=http://localhost:8002
-OCTOPOES_API=http://localhost:8001
+# KATALOGUS_API=http://localhost:8003
+# BYTES_API=http://localhost:8002
+# OCTOPOES_API=http://localhost:8001
 
 # Bytes specific api credentials
 BYTES_USERNAME=bytes

mula/packaging/deb/data/usr/bin/update-mula-db

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -ae
+source /usr/lib/kat/mula.defaults
 source /etc/kat/mula.conf
 cd /opt/venvs/kat-mula/lib/python*/site-packages
 /opt/venvs/kat-mula/bin/python -m alembic --config scheduler/alembic.ini upgrade head

mula/packaging/deb/data/usr/lib/kat/mula.defaults

@@ -0,0 +1,6 @@
+SCHEDULER_API_HOST=localhost
+SCHEDULER_API_PORT=8004
+SCHEDULER_LOG_CFG=/etc/kat/mula.logging.json
+KATALOGUS_API=http://localhost:8003
+BYTES_API=http://localhost:8002
+OCTOPOES_API=http://localhost:8001

mula/packaging/deb/data/usr/lib/systemd/system/kat-mula.service

@@ -7,7 +7,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-mula
 WorkingDirectory=/opt/venvs/kat-mula/
-# TODO: don't load env here. It exposes the values over dbus IPC
+EnvironmentFile=/usr/lib/kat/mula.defaults
 EnvironmentFile=/etc/kat/mula.conf
 ExecStart=/opt/venvs/kat-mula/bin/python -m scheduler
 Restart=on-failure

mula/scheduler/config/settings.py

@@ -31,7 +31,12 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]:
             # ...but old variable has been explicitly set through env
             if new_name not in env_vars and old_name in env_vars:
                 logging.warning("Deprecation: %s is deprecated, use %s instead", old_name.upper(), new_name.upper())
-                d[new_name[len(env_prefix) :]] = env_vars[old_name]
+                if new_name == "queue_uri":
+                    d["host_mutation"] = env_vars[old_name]
+                    d["host_raw_data"] = env_vars[old_name]
+                    d["host_normalizer_meta"] = env_vars[old_name]
+                else:
+                    d[new_name[len(env_prefix) :]] = env_vars[old_name]
 
         return d
 

octopoes/debian/install

@@ -1,2 +1,2 @@
 packaging/deb/data/etc/kat/* etc/kat
-packaging/deb/data/usr/lib/systemd/system/* usr/lib/systemd/system
+packaging/deb/data/usr/* usr

octopoes/packaging/deb/data/etc/kat/octopoes.conf

@@ -1,4 +1,4 @@
-XTDB_URI="http://localhost:3000"
+# XTDB_URI="http://localhost:3000"
 QUEUE_URI=
-OCTOPOES_LOG_CFG=/etc/kat/octopoes.logging.yml
-KATALOGUS_API=http://localhost:8003
+# OCTOPOES_LOG_CFG=/etc/kat/octopoes.logging.yml
+# KATALOGUS_API=http://localhost:8003

octopoes/packaging/deb/data/usr/lib/kat/octopoes.defaults

@@ -0,0 +1,3 @@
+XTDB_URI="http://localhost:3000"
+OCTOPOES_LOG_CFG=/etc/kat/octopoes.logging.yml
+KATALOGUS_API=http://localhost:8003

octopoes/packaging/deb/data/usr/lib/systemd/system/kat-octopoes-worker.service

@@ -7,6 +7,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-octopoes-worker
 WorkingDirectory=/opt/venvs/kat-octopoes
+EnvironmentFile=/usr/lib/kat/octopoes.defaults
 EnvironmentFile=/etc/kat/octopoes.conf
 ExecStart=/opt/venvs/kat-octopoes/bin/python -m celery -A octopoes.tasks.tasks worker -B -s /tmp/celerybeat-schedule --loglevel=WARNING
 Restart=on-failure

octopoes/packaging/deb/data/usr/lib/systemd/system/kat-octopoes.service

@@ -8,6 +8,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-octopoes
 WorkingDirectory=/opt/venvs/kat-octopoes
+EnvironmentFile=/usr/lib/kat/octopoes.defaults
 EnvironmentFile=/etc/kat/octopoes.conf
 ExecStart=/opt/venvs/kat-octopoes/bin/python -m gunicorn \
           --access-logfile - \

rocky/debian/install

@@ -1,4 +1,3 @@
 packaging/deb/data/etc/kat/* etc/kat
-packaging/deb/data/usr/bin/* usr/bin
-packaging/deb/data/usr/lib/systemd/system/kat-rocky.service usr/lib/systemd/system
+packaging/deb/data/usr/* usr
 OOI_database_seed.json usr/share/kat-rocky

rocky/packaging/deb/data/etc/kat/rocky.conf

@@ -1,7 +1,7 @@
 # Django settings
 SECRET_KEY=
-DEBUG="False"
-TWOFACTOR_ENABLED="True"
+# DEBUG="False"
+# TWOFACTOR_ENABLED="True"
 DJANGO_ALLOWED_HOSTS="localhost"
 DJANGO_CSRF_TRUSTED_ORIGINS="https://localhost"
 
@@ -16,11 +16,11 @@ ROCKY_DB_PORT=5432
 QUEUE_URI=
 
 # External services
-OCTOPOES_API=http://localhost:8001
-SCHEDULER_API=http://localhost:8004
-KATALOGUS_API=http://localhost:8003
-KEIKO_API=http://localhost:8005
-BYTES_API=http://localhost:8002
+# OCTOPOES_API=http://localhost:8001
+# SCHEDULER_API=http://localhost:8004
+# KATALOGUS_API=http://localhost:8003
+# KEIKO_API=http://localhost:8005
+# BYTES_API=http://localhost:8002
 BYTES_USERNAME=bytes
 BYTES_PASSWORD=
 

rocky/packaging/deb/data/usr/bin/rocky-cli

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -ae
+source /usr/lib/kat/rocky.defaults
 source /etc/kat/rocky.conf
 
 /opt/venvs/kat-rocky/bin/manage.py "$@"

rocky/packaging/deb/data/usr/lib/kat/rocky.defaults

@@ -0,0 +1,5 @@
+OCTOPOES_API=http://localhost:8001
+SCHEDULER_API=http://localhost:8004
+KATALOGUS_API=http://localhost:8003
+KEIKO_API=http://localhost:8005
+BYTES_API=http://localhost:8002

rocky/packaging/deb/data/usr/lib/systemd/system/kat-rocky.service

@@ -7,6 +7,7 @@ User=kat
 Group=kat
 SyslogIdentifier=kat-rocky
 WorkingDirectory=/opt/venvs/kat-rocky/
+EnvironmentFile=/usr/lib/kat/rocky.defaults
 EnvironmentFile=/etc/kat/rocky.conf
 ExecStart=/opt/venvs/kat-rocky/bin/uwsgi --ini /etc/kat/rocky.uwsgi.ini
 Restart=on-failure