diff --git a/.gitignore b/.gitignore index b21bcb6..9b1e900 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,4 @@ dmypy.json tmp/** .DS_Store test/** +.secrets diff --git a/.vscode/launch.json b/.vscode/launch.json index c94d226..b858ffe 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,7 +4,19 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - + { + "name": "Python Debugger: Current File with Arguments", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "duplicity-photo", + "photo-weekly/*.sigtar.gpg", + "--dry-run" + ], + "envFile": "${workspaceFolder}/.secrets" + }, { "name": "Python: Remote Attach", "type": "python", @@ -23,7 +35,6 @@ // python -m debugpy --listen 5678 --wait-for-client ./backup.py // python -m pip install debugpy // kubectl -n duplicity-backup --address 0.0.0.0 port-forward pods/duplicity-photo-weekly-debug 5678:5678 - }, { "name": "Python: backup.py", @@ -37,7 +48,8 @@ }, "envFile": "${workspaceFolder}/tmp/.env", "args": [ - "--config", "test/backup.yaml", + "--config", + "test/backup.yaml", // "./tmp/conf.yaml", // "--command", // "collection-status", diff --git a/Dockerfile b/Dockerfile index e0a88f1..e5e2a07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN apk fix && \ RUN pip install setuptools_scm boto3 python-gettext &&\ apk add gettext &&\ apk add -t .build-deps linux-headers python3-dev librsync-dev gcc musl-dev git &&\ - pip install https://gitlab.com/duplicity/duplicity/-/archive/dev/duplicity-dev.tar.gz &&\ + pip install https://gitlab.com/duplicity/duplicity/-/archive/issue831/duplicity-issue831.tar.gz &&\ apk del --purge .build-deps RUN addgroup -S app &&\ diff --git a/src/helper_initiate_glacier_restore.py b/src/helper_initiate_glacier_restore.py new file mode 100644 index 0000000..841be92 --- /dev/null +++ b/src/helper_initiate_glacier_restore.py @@ -0,0 +1,70 @@ +import time +import boto3 +import fnmatch +import argparse + +def wait(seconds): + print(f"Wait {seconds}s: ", end="") + for i in range(seconds): + print(".", end="", flush=True) + time.sleep(1) + print("") + +def initiate_restore(bucket_name, file_pattern, days=7, dry_run=False): + # Create a session using the default profile + session = boto3.Session() + + # Create an S3 client + s3 = session.client('s3') + + # List objects in the bucket + paginator = s3.get_paginator('list_objects_v2') + page_iterator = paginator.paginate(Bucket=bucket_name) + + + restored_files = 0 + + for page in page_iterator: + if 'Contents' in page: + for obj in page['Contents']: + key = obj['Key'] + if fnmatch.fnmatch(key, file_pattern) and obj['StorageClass'] == 'GLACIER': + response = s3.head_object(Bucket=bucket_name, Key=key) + trigger_restore = False + if 'Restore' in response: + restore_status = response['Restore'] + print(f"Restore status for {key}: {restore_status}") + if 'ongoing-request="true"' in restore_status: + print(f"{key} is currently being restored.") + elif 'ongoing-request="false"' in restore_status: + print(f"{key} has been successfully restored and is now available for access.") + else: + trigger_restore = True + restored_files += 1 + if not dry_run and trigger_restore: + response = s3.restore_object( + Bucket=bucket_name, + Key=key, + RestoreRequest={ + 'Days': days, + 'GlacierJobParameters': { + 'Tier': 'Expedited' # Use 'Bulk' or 'Expedited' for different retrieval options + } + } + ) + print(f"Restore initiated: {key}, Status: {response['ResponseMetadata']['HTTPStatusCode']}") + elif dry_run: + print(f"Dry run: trigger restore of {key}") + if restored_files % 3 == 0 and restored_files > 0: + wait(300) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Initiate restore on S3 Glacier Flexible Retrieval by file pattern.') + parser.add_argument('bucket_name', type=str, help='The name of the S3 bucket.') + parser.add_argument('file_pattern', type=str, help='The file pattern to match (e.g., "*.txt").') + parser.add_argument('--days', type=int, default=7, help='The number of days to keep the restored files available.') + parser.add_argument('--dry-run', action="store_true", help='show what would happen') + + args = parser.parse_args() + + initiate_restore(args.bucket_name, args.file_pattern, args.days, args.dry_run)