From eb340496d2daef2b81c1b888ae1cd9809e0a09bc Mon Sep 17 00:00:00 2001
From: Paul Scheunemann <33927916+ailox@users.noreply.github.com>
Date: Wed, 4 Dec 2024 16:01:50 +0100
Subject: [PATCH] Add upgrade scripts for database files (#28)

* Change: Add upgrade scripts for database files

Postgres changes the format of the database files on disk between major
versions. This means that in order to upgrade a postgres database, it is
not sufficient to just bump the container versions.

To help with this process, we now can use migration scripts that will
take care of updating the database version without the need of a manual
dump/restore.

* Update README with warning
---
 .dockerignore         |  4 +++
 Dockerfile            | 35 ++++++++++++++++++++++++-
 README.md             | 31 ++++++++++++++++++++++
 bin/upgradeversion.sh | 60 +++++++++++++++++++++++++++++++++++++++++++
 tests/test-upgrade.sh | 41 +++++++++++++++++++++++++++++
 5 files changed, 170 insertions(+), 1 deletion(-)
 create mode 100644 .dockerignore
 create mode 100755 bin/upgradeversion.sh
 create mode 100755 tests/test-upgrade.sh

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..63821da
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,4 @@
+.dockerignore
+LICENSE
+README.md
+tests/
diff --git a/Dockerfile b/Dockerfile
index 4280d6f..18196df 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,2 +1,35 @@
-ARG POSTGRES_VERSION=16
+# This image was forked from the official PostgreSQL image, so it could be
+# signed and relied on as an internal dependency.
+# The image is modified to include an older version of PostgreSQL, which
+# allows us to upgrade the database from the old version to the new one.
+ARG POSTGRES_VERSION=17
 FROM postgres:${POSTGRES_VERSION}
+
+# we need to redeclare the ARG here, otherwise it will not
+# be available in the section below the FROM statement.
+ARG POSTGRES_VERSION=17
+ARG POSTGRES_OLD_VERSION=16
+
+ENV POSTGRES_VERSION=${POSTGRES_VERSION}
+ENV POSTGRES_OLD_VERSION=${POSTGRES_OLD_VERSION}
+
+# Enable and install old version of PostgreSQL.
+RUN sed -i "s/\$/ ${POSTGRES_OLD_VERSION}/" /etc/apt/sources.list.d/pgdg.list
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		postgresql-${POSTGRES_OLD_VERSION} \
+	; \
+	rm -rf /var/lib/apt/lists/*
+
+# The old binaries will be in /usr/lib/postgresql/16/bin
+ENV PGBINOLD /usr/lib/postgresql/${POSTGRES_OLD_VERSION}/bin
+ENV PGBINNEW /usr/lib/postgresql/${POSTGRES_VERSION}/bin
+
+# we are usually using /var/lib/postgresql/data as the data directory
+# so this is why we are using it for the 'old' version instead of the
+# path that is customized for the version.
+ENV PGDATAOLD /var/lib/postgresql/data
+ENV PGDATANEW /var/lib/postgresql/${POSTGRES_VERSION}/data
+
+COPY bin/upgradeversion.sh /usr/local/bin/upgradeversion
diff --git a/README.md b/README.md
index cd66e81..bc0d2a3 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,37 @@
 
 Opensight-Postgres is utilized to provide our Opensight services with the appropriate PostgreSQL container versions.
 
+It is equivalent to the [postgres](https://hub.docker.com/_/postgres) Docker
+image. Additionally one previous major version of postgres is added to the
+image, so that it can be used to upgrade mounted database files to the
+current version.
+
+## Upgrade process
+The opensight-postgres image can be used to upgrade older database versions
+to the current version used by this image.
+
+If the database is already upgraded, this will be a no-op and the upgrade
+process will automatically terminate with exit code 0.
+
+This does not happen automatically however, since it represents a backwards
+incompatible change, and has potential to fail. It is therefore recommended
+to create backups before attempting a migration.
+
+Usage:
+
+```
+docker run \
+    -it --rm \
+    --user 999:999 \
+    -v /path/to/database:/var/lib/postgres/data/ \
+    opensight-postgres upgradeversion inplace
+```
+
+The inplace argument is required to perform the upgrade in the same mounted
+directory. This has the implication that the database will be corrupted if
+the upgrade fails. Therefore it is recommended to create a backup before
+attempting the upgrade.
+
 ## Maintainer
 
 This project is maintained by [Greenbone AG](https://www.greenbone.net/).
diff --git a/bin/upgradeversion.sh b/bin/upgradeversion.sh
new file mode 100755
index 0000000..163d671
--- /dev/null
+++ b/bin/upgradeversion.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# POSTGRES_VERSION, POSTGRES_OLD_VERSION
+# PGDATAOLD, PGDATANEW, PGBINOLD, PGBINNEW
+# are available from the Dockerfile
+
+# If there is no initialized database, we do nothing
+if [ ! -s "${PGDATAOLD}/PG_VERSION" ]; then
+    echo "No database found. Exiting."
+    exit 0
+fi
+
+# Check postgres version
+echo -n "Current database version is "
+cat ${PGDATAOLD}/PG_VERSION
+
+# if the version is already current, we don't need to do anything
+if [ "$(cat ${PGDATAOLD}/PG_VERSION)" == "${POSTGRES_VERSION}" ]; then
+    echo "No need to update. exiting."
+    exit 0
+fi
+
+# Abort if the old version is not supported
+if [ "$(cat ${PGDATAOLD}/PG_VERSION)" != "${POSTGRES_OLD_VERSION}" ]; then
+    echo "Only PostgreSQL ${POSTGRES_OLD_VERSION} is supported for migration with this image."
+    exit 1
+fi
+
+# Initialize the new database, if it doesn't exist
+if [ ! -s "${PGDATANEW}/PG_VERSION" ]; then
+    initdb -D ${PGDATANEW} -U "${POSTGRES_USER}"
+
+		# migrate auth settings as well
+		cp --target-directory=${PGDATANEW}/ ${PGDATAOLD}/pg_hba.conf
+fi
+
+# mitigate: 'you must have read and write access in the current directory'
+cd /var/lib/postgresql
+
+pg_upgrade \
+    -U "${POSTGRES_USER}" \
+    --old-bindir ${PGBINOLD} \
+    --new-bindir ${PGBINNEW} \
+    --old-datadir ${PGDATAOLD} \
+    --new-datadir ${PGDATANEW}
+
+# Check postgres version
+echo -n "New migrated database version is "
+cat ${PGDATANEW}/PG_VERSION
+
+# if first parameter is 'inplace', we copy the new postgres version to
+# the old one. This is not an atomic operation, and therefore risky;
+# If the copy operation is not successful, we will end up with a
+# corrupted database.
+if [ "${1:-}" == "inplace" ]; then
+    rm -rf ${PGDATAOLD}/*
+    mv --target-directory=${PGDATAOLD}/ ${PGDATANEW}/*
+fi
diff --git a/tests/test-upgrade.sh b/tests/test-upgrade.sh
new file mode 100755
index 0000000..490b727
--- /dev/null
+++ b/tests/test-upgrade.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script is used to test the upgradeversion script without spinning up the
+# whole OpenSight stack. It creates a new (outdated) PostgreSQL database,
+# attempts to upgrade it and checks if the version file was updated correctly.
+
+DOCKER_IMAGE=packages.greenbone.net/opensight/opensight-postgres
+PREVIOUS_VERSION=16.6  # released version to test the upgrade from
+
+tempdir=$(mktemp -d -t opensight-postgres-test-XXXXX)
+
+cleanup() {
+  sudo rm -rf "${tempdir}"
+}
+trap cleanup ERR
+
+sudo chown -R 999:999 "${tempdir}"
+
+# Start a PostgreSQL container with an outdated version
+docker run -it --rm \
+  --user 999:999 \
+  -e POSTGRES_PASSWORD=password \
+  -e POSTGRES_DB=keycloak \
+  -e POSTGRES_USER=keycloak \
+  -v "${tempdir}":/var/lib/postgresql/data \
+  ${DOCKER_IMAGE}:${PREVIOUS_VERSION}
+
+# Upgrade the database with the test version
+docker build -t ${DOCKER_IMAGE}:test .
+docker run -it --rm \
+  --user 999:999 -e POSTGRES_PASSWORD=password \
+  -e POSTGRES_DB=keycloak \
+  -e POSTGRES_USER=keycloak \
+  -v "${tempdir}":/var/lib/postgresql/data \
+  ${DOCKER_IMAGE}:test upgradeversion inplace
+
+# did everything work as expected?
+sudo cat ${tempdir}/PG_VERSION
+
+cleanup