feat(s3): add s3 support for backup storage

Dockerfile

@@ -7,5 +7,8 @@ COPY backup.sh /usr/local/bin/backup.sh
 
 RUN microdnf update -y && rm -rf /var/cache/yum
 RUN microdnf install findutils -y && microdnf clean all
+RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc.rpm \
+ && rpm -ih mc.rpm \
+ && rm mc.rpm
 
 CMD ["/usr/local/bin/backup.sh"]

README.md

@@ -59,6 +59,12 @@ oc edit -n etcd-backup cm/backup-config
 ```
 
 The following options are used:
+- `OCP_BACKUP_S3`: Use S3 to store etcd-backup snapshots
+- `OCP_BACKUP_S3_NAME`: MinIO client host alias name
+- `OCP_BACKUP_S3_HOST`: S3 host endpoint (with scheme)
+- `OCP_BACKUP_S3_BUCKET`: S3 bucket name
+- `OCP_BACKUP_S3_ACCESS_KEY`: access key to access S3 bucket
+- `OCP_BACKUP_S3_SECRET_KEY`: secret key to access S3 bucket
 - `OCP_BACKUP_SUBDIR`: Sub directory on PVC that should be used to store the backup. If it does not exist it will be created.
 - `OCP_BACKUP_DIRNAME`: Directory name for a single backup. This is a format string used by
 [`date`](https://man7.org/linux/man-pages/man1/date.1.html)
@@ -70,6 +76,8 @@ The following options are used:
 - `OCP_BACKUP_KEEP_COUNT`: Number of backups to keep. Only used if `backup.expiretype` is set to `count`
 - `OCP_BACKUP_UMASK`: Umask used inside the script to set restrictive permission on written files, as they contain sensitive information.
 
+Note that the storage type is exclusive. This means it is either S3 or PVC. In case of using S3 we do not manage the retention within the backup script. We suggest using a rentention policy on the S3 bucket itself. This can be done thanks to an objects expiration configuration as described in the object lifecycle management [documentation](https://min.io/docs/minio/linux/administration/object-management/object-lifecycle-management.html#object-expiration).
+
 Changing the schedule be done in the CronJob directly, with `spec.schedule`:
 ```
 oc edit -n etcd-backup cronjob/etcd-backup

backup-config.yaml

@@ -3,6 +3,12 @@ apiVersion: v1
 metadata:
   name: backup-config
 data:
+  OCP_BACKUP_S3: "false"
+  OCP_BACKUP_S3_NAME: "minio"
+  OCP_BACKUP_S3_HOST: "http://minio.local:9000"
+  OCP_BACKUP_S3_BUCKET: "etcd-backup"
+  OCP_BACKUP_S3_ACCESS_KEY: "randomaccesskey"
+  OCP_BACKUP_S3_SECRET_KEY: "secretkey"
   OCP_BACKUP_SUBDIR: "/"
   OCP_BACKUP_DIRNAME: "+etcd-backup-%FT%T%:z"
   OCP_BACKUP_EXPIRE_TYPE: "days"

backup.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
+
 ################################################################################
 # backup.sh OpenShift etcd backup script
 ################################################################################
 #
-# Copyright (C) 2021 Adfinis AG
+# Copyright (C) 2024 Adfinis AG
 #                    https://adfinis.com
 #                    [email protected]
 #
@@ -26,53 +27,75 @@
 #
 # Authors:
 #  Cyrill von Wattenwyl <[email protected]>
-
+#  Valentin Maillot <[email protected]>
 
 set -xeuo pipefail
 
-# set proper umask
-umask "${OCP_BACKUP_UMASK}"
+# check storage type
+if [ "${OCP_BACKUP_S3}" = "true" ]; then
+    # prepare & push backup to S3
 
-# validate expire type
-case "${OCP_BACKUP_EXPIRE_TYPE}" in
-    days|count|never) ;;
-    *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;;
-esac
+    # update CA trust
+    update-ca-trust
 
-# validate  expire numbers
-if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then
-  case "${OCP_BACKUP_KEEP_DAYS}" in
-    ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;;
-    *) ;;
-  esac
-elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then
-  case "${OCP_BACKUP_KEEP_COUNT}" in
-    ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;;
-    *) ;;
-  esac
-fi
+    # configure mcli assuming the bucket already exists
+    bash +o history
+    mcli alias set "${OCP_BACKUP_S3_NAME}" "${OCP_BACKUP_S3_HOST}" "${OCP_BACKUP_S3_ACCESS_KEY}" "${OCP_BACKUP_S3_SECRET_KEY}"
+    bash -o history
+
+    # create backup to temporary location
+    chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup
+
+    # move files to S3 and delete temporary files
+    mcli mv /host/var/tmp/etcd-backup/* "${OCP_BACKUP_S3_NAME}"/"${OCP_BACKUP_S3_BUCKET}"
+    rm -rv /host/var/tmp/etcd-backup
+else
+    # prepare, run and copy backup
+
+    # set proper umask
+    umask "${OCP_BACKUP_UMASK}"
+
+    # validate expire type
+    case "${OCP_BACKUP_EXPIRE_TYPE}" in
+        days|count|never) ;;
+        *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;;
+    esac
+
+    # validate expire numbers
+    if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then
+      case "${OCP_BACKUP_KEEP_DAYS}" in
+        ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;;
+        *) ;;
+      esac
+    elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then
+      case "${OCP_BACKUP_KEEP_COUNT}" in
+        ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;;
+        *) ;;
+      esac
+    fi
 
-# make dirname and cleanup paths
-BACKUP_FOLDER="$( date "${OCP_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; }
-BACKUP_PATH="$( realpath -m "${OCP_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )"
-BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )"
-BACKUP_ROOTPATH="$( realpath -m "/backup/${OCP_BACKUP_SUBDIR}" )"
+    # make dirname and cleanup paths
+    BACKUP_FOLDER="$( date "${OCP_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; }
+    BACKUP_PATH="$( realpath -m "${OCP_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )"
+    BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )"
+    BACKUP_ROOTPATH="$( realpath -m "/backup/${OCP_BACKUP_SUBDIR}" )"
 
-# make nescesary directorys
-mkdir -p "/host/var/tmp/etcd-backup"
-mkdir -p "${BACKUP_PATH_POD}"
+    # make necessary directories
+    mkdir -p "/host/var/tmp/etcd-backup"
+    mkdir -p "${BACKUP_PATH_POD}"
 
-# create backup to temporary location
-chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup
+    # create backup to temporary location
+    chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup
 
-# move files to pvc and delete temporary files
-mv /host/var/tmp/etcd-backup/* "${BACKUP_PATH_POD}"
-rm -rv /host/var/tmp/etcd-backup
+    # move files to PVC and delete temporary files
+    mv /host/var/tmp/etcd-backup/* "${BACKUP_PATH_POD}"
+    rm -rv /host/var/tmp/etcd-backup
 
-# expire backup
-if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then
-  find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1  -type d -mtime "+${OCP_BACKUP_KEEP_DAYS}" -exec rm -rv {} +
-elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then
-  # shellcheck disable=SC3040,SC2012
-  ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${OCP_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}"
+    # expire backup
+    if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then
+      find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1  -type d -mtime "+${OCP_BACKUP_KEEP_DAYS}" -exec rm -rv {} +
+    elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then
+      # shellcheck disable=SC3040,SC2012
+      ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${OCP_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}"
+    fi
 fi