Add aws support

Obmondo · Mar 19, 2024 · 9212b41 · 9212b41
1 parent 3a5da7f
commit 9212b41
Showing 1 changed file with 38 additions and 12 deletions.
diff --git a/logical-backup/dump.sh b/logical-backup/dump.sh
@@ -13,6 +13,7 @@ ERRORCOUNT=0
 POSTGRES_OPERATOR=spilo
 LOGICAL_BACKUP_PROVIDER=${LOGICAL_BACKUP_PROVIDER:="s3"}
 LOGICAL_BACKUP_S3_RETENTION_TIME=${LOGICAL_BACKUP_S3_RETENTION_TIME:=""}
+LOGICAL_BACKUP_AZURE_RETENTION_TIME=${LOGICAL_BACKUP_AZURE_RETENTION_TIME:=""}
 
 PG_DUMP_EXTRA_ARGUMENTS=${PG_DUMP_EXTRA_ARGUMENTS:=""}
 PG_DUMPALL_EXTRA_ARGUMENTS=${PG_DUMPALL_EXTRA_ARGUMENTS:=""}
@@ -43,9 +44,9 @@ function compress {
 }
 
 function az_upload {
-  PATH_TO_BACKUP="${LOGICAL_BACKUP_S3_BUCKET}/${POSTGRES_OPERATOR}/${LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX}/logical_backups/$(date +%s).sql.gz"
+  PATH_TO_BACKUP="${LOGICAL_BACKUP_AZURE_CONTAINER}/${POSTGRES_OPERATOR}/${LOGICAL_BACKUP_AZURE_SCOPE_SUFFIX}/logical_backups/$(date +%s).sql.gz"
 
-  az storage blob upload --file "${1}" --account-name "${LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME}" --account-key "${LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY}" -c "${LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER}" -n "${PATH_TO_BACKUP}"
+  az storage blob upload --file "${1}" --account-name "${LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME}" --account-key "${LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY}" -c "${LOGICAL_BACKUP_AZURE_CONTAINER}" -n "${PATH_TO_BACKUP}"
 }
 
 function aws_delete_objects {
@@ -105,7 +106,7 @@ function aws_upload {
   # mimic bucket setup from Spilo
   # to keep logical backups at the same path as WAL
   # NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator
-  PATH_TO_BACKUP=s3://${LOGICAL_BACKUP_S3_BUCKET}"/"${POSTGRES_OPERATOR}"/"${LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX}"/logical_backups/"$(date +%s).sql.gz
+  PATH_TO_BACKUP="s3://${LOGICAL_BACKUP_S3_BUCKET}/${POSTGRES_OPERATOR}/${LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX}/logical_backups/$(date +%s).sql.gz"
 
   args=()
 
@@ -132,26 +133,51 @@ function upload {
       aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
       aws_delete_outdated
       ;;
+    "az")
+      az_upload
+      ;;
   esac
 }
 
-if [ "$LOGICAL_BACKUP_PROVIDER" == "az" && "$PGDATABASE" == "az" ]; then
-PATH_TO_BACKUP="${POSTGRES_OPERATOR}/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%Y-%m-%dT%H%M%S)
+if [[ "$LOGICAL_BACKUP_PROVIDER" == "s3" && "$PGDATABASE" == "s3" ]]; then
+    PATH_TO_BACKUP="s3://${LOGICAL_BACKUP_S3_BUCKET}/${POSTGRES_OPERATOR}/${LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX}/logical_backups/$(date +%Y-%m-%dT%H%M%S)"
+
+    echo "Dumping and uploading global items..."
+    dump_global | compress | aws_upload "$PATH_TO_BACKUP/global.sql.gz"
+
+    [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
+    set +x
+
+    list_databases > /tmp/database-list
+    while read dbname; do 
+        echo "Dumping $dbname..."
+        dump_db $dbname
+        echo "Uploading directory /tmp/db-$dbname to $PATH_TO_BACKUP/$dbname..."
+        aws_upload "/tmp/db-$dbname" "$PATH_TO_BACKUP/$dbname"
+        echo "Cleaning up /tmp/db-$dbname..."
+        rm -rf "/tmp/db-$dbname"
+    done < /tmp/database-list
+
+    rm /tmp/database-list
+
+    exit $ERRORCOUNT
+elif [[ "$LOGICAL_BACKUP_PROVIDER" == "az" && "$PGDATABASE" == "az" ]]; then
+    PATH_TO_BACKUP="${POSTGRES_OPERATOR}/"$SCOPE$LOGICAL_BACKUP_AZURE_SCOPE_SUFFIX"/logical_backups/"$(date +%Y-%m-%dT%H%M%S)
 
-    echo Dumping and uploading global items...
-    dump_global | compress | az_upload $PATH_TO_BACKUP/global.sql.gz
+    echo "Dumping and uploading global items..."
+    dump_global | compress | az_upload "$PATH_TO_BACKUP/global.sql.gz"
 
     [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
     set +x
 
     list_databases > /tmp/database-list
     while read dbname; do 
-        echo Dumping $dbname...
+        echo "Dumping $dbname..."
         dump_db $dbname
-        echo Uploading directory /tmp/db-$dbname to $PATH_TO_BACKUP/$dbname...
-        az_upload_dir /tmp/db-$dbname $PATH_TO_BACKUP/$dbname
-        echo Cleaning up /tmp/db-$dbname...
-        rm -rf /tmp/db-$dbname
+        echo "Uploading directory /tmp/db-$dbname to $PATH_TO_BACKUP/$dbname..."
+        az_upload "/tmp/db-$dbname" "$PATH_TO_BACKUP/$dbname"
+        echo "Cleaning up /tmp/db-$dbname..."
+        rm -rf "/tmp/db-$dbname"
     done < /tmp/database-list
 
     rm /tmp/database-list