From a5f40b983299e5de05f991202cf8c86fdfc67a80 Mon Sep 17 00:00:00 2001 From: Valentin Maillot Date: Tue, 23 Jan 2024 17:25:58 +0100 Subject: [PATCH 1/4] feat(s3): add s3 support for backup storage Signed-off-by: Valentin Maillot --- Dockerfile | 1 + README.md | 8 ++++ backup-config.yaml | 6 +++ backup.sh | 103 +++++++++++++++++++++++++++------------------ 4 files changed, 78 insertions(+), 40 deletions(-) diff --git a/Dockerfile b/Dockerfile index fcf0c1c..1c576e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,5 +7,6 @@ COPY backup.sh /usr/local/bin/backup.sh RUN microdnf update -y && rm -rf /var/cache/yum RUN microdnf install findutils -y && microdnf clean all +RUN curl -s https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc CMD ["/usr/local/bin/backup.sh"] diff --git a/README.md b/README.md index 3168a28..193720c 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,12 @@ oc edit -n etcd-backup cm/backup-config ``` The following options are used: +- `OCP_BACKUP_S3`: Use S3 to store etcd-backup snapshots +- `OCP_BACKUP_S3_NAME`: MinIO client host alias name +- `OCP_BACKUP_S3_HOST`: S3 host endpoint (with scheme) +- `OCP_BACKUP_S3_BUCKET`: S3 bucket name +- `OCP_BACKUP_S3_ACCESS_KEY`: access key to access S3 bucket +- `OCP_BACKUP_S3_SECRET_KEY`: secret key to access S3 bucket - `OCP_BACKUP_SUBDIR`: Sub directory on PVC that should be used to store the backup. If it does not exist it will be created. - `OCP_BACKUP_DIRNAME`: Directory name for a single backup. This is a format string used by [`date`](https://man7.org/linux/man-pages/man1/date.1.html) @@ -70,6 +76,8 @@ The following options are used: - `OCP_BACKUP_KEEP_COUNT`: Number of backups to keep. Only used if `backup.expiretype` is set to `count` - `OCP_BACKUP_UMASK`: Umask used inside the script to set restrictive permission on written files, as they contain sensitive information. +Note that the storage type is exclusive. This means it is either S3 or PVC. In case of using S3 we do not manage the retention within the backup script. We suggest using a rentention policy on the S3 bucket itself. + Changing the schedule be done in the CronJob directly, with `spec.schedule`: ``` oc edit -n etcd-backup cronjob/etcd-backup diff --git a/backup-config.yaml b/backup-config.yaml index 73e142b..59edffe 100644 --- a/backup-config.yaml +++ b/backup-config.yaml @@ -3,6 +3,12 @@ apiVersion: v1 metadata: name: backup-config data: + OCP_BACKUP_S3: "false" + OCP_BACKUP_S3_NAME: "minio" + OCP_BACKUP_S3_HOST: "http://minio.local:9000" + OCP_BACKUP_S3_BUCKET: "etcd-backup" + OCP_BACKUP_S3_ACCESS_KEY: "randomaccesskey" + OCP_BACKUP_S3_SECRET_KEY: "secretkey" OCP_BACKUP_SUBDIR: "/" OCP_BACKUP_DIRNAME: "+etcd-backup-%FT%T%:z" OCP_BACKUP_EXPIRE_TYPE: "days" diff --git a/backup.sh b/backup.sh index 024a928..aade772 100755 --- a/backup.sh +++ b/backup.sh @@ -1,9 +1,10 @@ #!/bin/bash + ################################################################################ # backup.sh OpenShift etcd backup script ################################################################################ # -# Copyright (C) 2021 Adfinis AG +# Copyright (C) 2024 Adfinis AG # https://adfinis.com # info@adfinis.com # @@ -26,53 +27,75 @@ # # Authors: # Cyrill von Wattenwyl - +# Valentin Maillot set -xeuo pipefail -# set proper umask -umask "${OCP_BACKUP_UMASK}" +# check storage type +if [ "${OCP_BACKUP_S3}" = "true" ]; then + # prepare & push backup to S3 -# validate expire type -case "${OCP_BACKUP_EXPIRE_TYPE}" in - days|count|never) ;; - *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;; -esac + # update CA trust + update-ca-trust -# validate expire numbers -if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then - case "${OCP_BACKUP_KEEP_DAYS}" in - ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;; - *) ;; - esac -elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then - case "${OCP_BACKUP_KEEP_COUNT}" in - ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;; - *) ;; - esac -fi + # configure mc assuming the bucket already exists + bash +o history + mc alias set "${OCP_BACKUP_S3_NAME}" "${OCP_BACKUP_S3_HOST}" "${OCP_BACKUP_S3_ACCESS_KEY}" "${OCP_BACKUP_S3_SECRET_KEY}" + bash -o history + + # create backup to temporary location + chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup + + # move files to S3 and delete temporary files + mc mv /host/var/tmp/etcd-backup/* "${OCP_BACKUP_S3_NAME}"/"${OCP_BACKUP_S3_BUCKET}" + rm -rv /host/var/tmp/etcd-backup +else + # prepare, run and copy backup + + # set proper umask + umask "${OCP_BACKUP_UMASK}" + + # validate expire type + case "${OCP_BACKUP_EXPIRE_TYPE}" in + days|count|never) ;; + *) echo "backup.expiretype needs to be one of: days,count,never"; exit 1 ;; + esac + + # validate expire numbers + if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then + case "${OCP_BACKUP_KEEP_DAYS}" in + ''|*[!0-9]*) echo "backup.expiredays needs to be a valid number"; exit 1 ;; + *) ;; + esac + elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then + case "${OCP_BACKUP_KEEP_COUNT}" in + ''|*[!0-9]*) echo "backup.expirecount needs to be a valid number"; exit 1 ;; + *) ;; + esac + fi -# make dirname and cleanup paths -BACKUP_FOLDER="$( date "${OCP_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; } -BACKUP_PATH="$( realpath -m "${OCP_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )" -BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )" -BACKUP_ROOTPATH="$( realpath -m "/backup/${OCP_BACKUP_SUBDIR}" )" + # make dirname and cleanup paths + BACKUP_FOLDER="$( date "${OCP_BACKUP_DIRNAME}")" || { echo "Invalid backup.dirname" && exit 1; } + BACKUP_PATH="$( realpath -m "${OCP_BACKUP_SUBDIR}/${BACKUP_FOLDER}" )" + BACKUP_PATH_POD="$( realpath -m "/backup/${BACKUP_PATH}" )" + BACKUP_ROOTPATH="$( realpath -m "/backup/${OCP_BACKUP_SUBDIR}" )" -# make nescesary directorys -mkdir -p "/host/var/tmp/etcd-backup" -mkdir -p "${BACKUP_PATH_POD}" + # make necessary directories + mkdir -p "/host/var/tmp/etcd-backup" + mkdir -p "${BACKUP_PATH_POD}" -# create backup to temporary location -chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup + # create backup to temporary location + chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup -# move files to pvc and delete temporary files -mv /host/var/tmp/etcd-backup/* "${BACKUP_PATH_POD}" -rm -rv /host/var/tmp/etcd-backup + # move files to PVC and delete temporary files + mv /host/var/tmp/etcd-backup/* "${BACKUP_PATH_POD}" + rm -rv /host/var/tmp/etcd-backup -# expire backup -if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then - find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1 -type d -mtime "+${OCP_BACKUP_KEEP_DAYS}" -exec rm -rv {} + -elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then - # shellcheck disable=SC3040,SC2012 - ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${OCP_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}" + # expire backup + if [ "${OCP_BACKUP_EXPIRE_TYPE}" = "days" ]; then + find "${BACKUP_ROOTPATH}" -mindepth 1 -maxdepth 1 -type d -mtime "+${OCP_BACKUP_KEEP_DAYS}" -exec rm -rv {} + + elif [ "${OCP_BACKUP_EXPIRE_TYPE}" = "count" ]; then + # shellcheck disable=SC3040,SC2012 + ls -1tp "${BACKUP_ROOTPATH}" | awk "NR>${OCP_BACKUP_KEEP_COUNT}" | xargs -I{} rm -rv "${BACKUP_ROOTPATH}/{}" + fi fi From a44e9be5498ee344c7a9518f610937c02b8c9242 Mon Sep 17 00:00:00 2001 From: vmaillot <74190001+vmaillot@users.noreply.github.com> Date: Wed, 24 Jan 2024 09:48:58 +0100 Subject: [PATCH 2/4] feat(Dockerfile): use mc RPM instead of binary directly Co-authored-by: Lucas Bickel <116588+hairmare@users.noreply.github.com> --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1c576e8..059659c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,8 @@ COPY backup.sh /usr/local/bin/backup.sh RUN microdnf update -y && rm -rf /var/cache/yum RUN microdnf install findutils -y && microdnf clean all -RUN curl -s https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc +RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc.rpm \ + && rpm -ih mc.rpm \ + && rm mc.rpm CMD ["/usr/local/bin/backup.sh"] From 973088eb2f6a11fa3af90312b60af3a876b93044 Mon Sep 17 00:00:00 2001 From: Valentin Maillot Date: Wed, 24 Jan 2024 09:51:56 +0100 Subject: [PATCH 3/4] feat(backup.sh): use mcli instead of mc Signed-off-by: Valentin Maillot --- backup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backup.sh b/backup.sh index aade772..a553e5a 100755 --- a/backup.sh +++ b/backup.sh @@ -38,16 +38,16 @@ if [ "${OCP_BACKUP_S3}" = "true" ]; then # update CA trust update-ca-trust - # configure mc assuming the bucket already exists + # configure mcli assuming the bucket already exists bash +o history - mc alias set "${OCP_BACKUP_S3_NAME}" "${OCP_BACKUP_S3_HOST}" "${OCP_BACKUP_S3_ACCESS_KEY}" "${OCP_BACKUP_S3_SECRET_KEY}" + mcli alias set "${OCP_BACKUP_S3_NAME}" "${OCP_BACKUP_S3_HOST}" "${OCP_BACKUP_S3_ACCESS_KEY}" "${OCP_BACKUP_S3_SECRET_KEY}" bash -o history # create backup to temporary location chroot /host /usr/local/bin/cluster-backup.sh /var/tmp/etcd-backup # move files to S3 and delete temporary files - mc mv /host/var/tmp/etcd-backup/* "${OCP_BACKUP_S3_NAME}"/"${OCP_BACKUP_S3_BUCKET}" + mcli mv /host/var/tmp/etcd-backup/* "${OCP_BACKUP_S3_NAME}"/"${OCP_BACKUP_S3_BUCKET}" rm -rv /host/var/tmp/etcd-backup else # prepare, run and copy backup From fa27ac8785a021a04cae4daaca2cfda362330f2d Mon Sep 17 00:00:00 2001 From: Valentin Maillot Date: Fri, 26 Jan 2024 13:24:00 +0100 Subject: [PATCH 4/4] chore(README): add link to MinIO object expiration configuration doc Signed-off-by: Valentin Maillot --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 193720c..8671462 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ The following options are used: - `OCP_BACKUP_KEEP_COUNT`: Number of backups to keep. Only used if `backup.expiretype` is set to `count` - `OCP_BACKUP_UMASK`: Umask used inside the script to set restrictive permission on written files, as they contain sensitive information. -Note that the storage type is exclusive. This means it is either S3 or PVC. In case of using S3 we do not manage the retention within the backup script. We suggest using a rentention policy on the S3 bucket itself. +Note that the storage type is exclusive. This means it is either S3 or PVC. In case of using S3 we do not manage the retention within the backup script. We suggest using a rentention policy on the S3 bucket itself. This can be done thanks to an objects expiration configuration as described in the object lifecycle management [documentation](https://min.io/docs/minio/linux/administration/object-management/object-lifecycle-management.html#object-expiration). Changing the schedule be done in the CronJob directly, with `spec.schedule`: ```