From 8e8b2396358f390f1a5548b903ebdc740f5604f3 Mon Sep 17 00:00:00 2001 From: vmaillot <74190001+vmaillot@users.noreply.github.com> Date: Mon, 26 Feb 2024 13:58:27 +0100 Subject: [PATCH] feat(s3): add s3 support for backup storage (#70) * feat(s3): add s3 support for backup storage * feat(Dockerfile): use mc RPM instead of binary directly * feat(backup.sh): use mcli instead of mc * chore(README): add link to MinIO object expiration configuration doc Signed-off-by: Valentin Maillot --------- Signed-off-by: Valentin Maillot Co-authored-by: Lucas Bickel <116588+hairmare@users.noreply.github.com> --- Dockerfile | 3 ++ README.md | 8 ++++ backup-config.yaml | 6 +++ backup.sh | 103 +++++++++++++++++++++++++++------------------ 4 files changed, 80 insertions(+), 40 deletions(-) diff --git a/Dockerfile b/Dockerfile index 248c875..6369b46 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,5 +7,8 @@ COPY backup.sh /usr/local/bin/backup.sh RUN microdnf update -y && rm -rf /var/cache/yum RUN microdnf install findutils -y && microdnf clean all +RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc.rpm \ + && rpm -ih mc.rpm \ + && rm mc.rpm CMD ["/usr/local/bin/backup.sh"] diff --git a/README.md b/README.md index 3168a28..8671462 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,12 @@ oc edit -n etcd-backup cm/backup-config ``` The following options are used: +- `OCP_BACKUP_S3`: Use S3 to store etcd-backup snapshots +- `OCP_BACKUP_S3_NAME`: MinIO client host alias name +- `OCP_BACKUP_S3_HOST`: S3 host endpoint (with scheme) +- `OCP_BACKUP_S3_BUCKET`: S3 bucket name +- `OCP_BACKUP_S3_ACCESS_KEY`: access key to access S3 bucket +- `OCP_BACKUP_S3_SECRET_KEY`: secret key to access S3 bucket - `OCP_BACKUP_SUBDIR`: Sub directory on PVC that should be used to store the backup. If it does not exist it will be created. - `OCP_BACKUP_DIRNAME`: Directory name for a single backup. This is a format string used by [`date`](https://man7.org/linux/man-pages/man1/date.1.html) @@ -70,6 +76,8 @@ The following options are used: - `OCP_BACKUP_KEEP_COUNT`: Number of backups to keep. Only used if `backup.expiretype` is set to `count` - `OCP_BACKUP_UMASK`: Umask used inside the script to set restrictive permission on written files, as they contain sensitive information. +Note that the storage type is exclusive. This means it is either S3 or PVC. In case of using S3 we do not manage the retention within the backup script. We suggest using a rentention policy on the S3 bucket itself. This can be done thanks to an objects expiration configuration as described in the object lifecycle management [documentation](https://min.io/docs/minio/linux/administration/object-management/object-lifecycle-management.html#object-expiration). + Changing the schedule be done in the CronJob directly, with `spec.schedule`: ``` oc edit -n etcd-backup cronjob/etcd-backup diff --git a/backup-config.yaml b/backup-config.yaml index 73e142b..59edffe 100644 --- a/backup-config.yaml +++ b/backup-config.yaml @@ -3,6 +3,12 @@ apiVersion: v1 metadata: name: backup-config data: + OCP_BACKUP_S3: "false" + OCP_BACKUP_S3_NAME: "minio" + OCP_BACKUP_S3_HOST: "http://minio.local:9000" + OCP_BACKUP_S3_BUCKET: "etcd-backup" + OCP_BACKUP_S3_ACCESS_KEY: "randomaccesskey" + OCP_BACKUP_S3_SECRET_KEY: "secretkey" OCP_BACKUP_SUBDIR: "/" OCP_BACKUP_DIRNAME: "+etcd-backup-%FT%T%:z" OCP_BACKUP_EXPIRE_TYPE: "days" diff --git a/backup.sh b/backup.sh index 024a928..a553e5a 100755 --- a/backup.sh +++ b/backup.sh @@ -1,9 +1,10 @@ #!/bin/bash + ################################################################################ # backup.sh OpenShift etcd backup script ################################################################################ # -# Copyright (C) 2021 Adfinis AG +# Copyright (C) 2024 Adfinis AG # https://adfinis.com # info@adfinis.com # @@ -26,53 +27,75 @@ # # Authors: # Cyrill von Wattenwyl - +# Valentin Maillot set -xeuo pipefail -# set proper umask -umask "${OCP_BACKUP_UMASK}" +# check storage type +if [ "${OCP_BACKUP_S3}" = "true" ]; then + # prepare & push backup to S3 -# validate expire type -case "${OCP_BACKUP_EXPIRE_TYPE}" in - days|count|never) ;; - *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;; -esac + # update CA trust + update-ca-trust -# validate expire numbers -if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then - case "${OCP_BACKUP_KEEP_DAYS}" in - ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;; - *) ;; - esac -elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then - case "${OCP_BACKUP_KEEP_COUNT}" in - ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;; - *) ;; - esac -fi + # configure mcli assuming the bucket already exists + bash +o history + mcli alias set "${OCP_BACKUP_S3_NAME}" "${OCP_BACKUP_S3_HOST}" "${OCP_BACKUP_S3_ACCESS_KEY}" "${OCP_BACKUP_S3_SECRET_KEY}" + bash -o history + + # create backup to temporary location + chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup + + # move files to S3 and delete temporary files + mcli mv /host/var/tmp/etcd-backup/* "${OCP_BACKUP_S3_NAME}"/"${OCP_BACKUP_S3_BUCKET}" + rm -rv /host/var/tmp/etcd-backup +else + # prepare, run and copy backup + + # set proper umask + umask "${OCP_BACKUP_UMASK}" + + # validate expire type + case "${OCP_BACKUP_EXPIRE_TYPE}" in + days|count|never) ;; + *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;; + esac + + # validate expire numbers + if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then + case "${OCP_BACKUP_KEEP_DAYS}" in + ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;; + *) ;; + esac + elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then + case "${OCP_BACKUP_KEEP_COUNT}" in + ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;; + *) ;; + esac + fi -# make dirname and cleanup paths -BACKUP_FOLDER="$( date "${OCP_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; } -BACKUP_PATH="$( realpath -m "${OCP_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )" -BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )" -BACKUP_ROOTPATH="$( realpath -m "/backup/${OCP_BACKUP_SUBDIR}" )" + # make dirname and cleanup paths + BACKUP_FOLDER="$( date "${OCP_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; } + BACKUP_PATH="$( realpath -m "${OCP_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )" + BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )" + BACKUP_ROOTPATH="$( realpath -m "/backup/${OCP_BACKUP_SUBDIR}" )" -# make nescesary directorys -mkdir -p "/host/var/tmp/etcd-backup" -mkdir -p "${BACKUP_PATH_POD}" + # make necessary directories + mkdir -p "/host/var/tmp/etcd-backup" + mkdir -p "${BACKUP_PATH_POD}" -# create backup to temporary location -chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup + # create backup to temporary location + chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup -# move files to pvc and delete temporary files -mv /host/var/tmp/etcd-backup/* "${BACKUP_PATH_POD}" -rm -rv /host/var/tmp/etcd-backup + # move files to PVC and delete temporary files + mv /host/var/tmp/etcd-backup/* "${BACKUP_PATH_POD}" + rm -rv /host/var/tmp/etcd-backup -# expire backup -if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then - find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1 -type d -mtime "+${OCP_BACKUP_KEEP_DAYS}" -exec rm -rv {} + -elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then - # shellcheck disable=SC3040,SC2012 - ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${OCP_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}" + # expire backup + if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then + find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1 -type d -mtime "+${OCP_BACKUP_KEEP_DAYS}" -exec rm -rv {} + + elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then + # shellcheck disable=SC3040,SC2012 + ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${OCP_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}" + fi fi