From 4615cc41b0e51ad68325e54ad30b5e7d90435772 Mon Sep 17 00:00:00 2001 From: Scanner Date: Mon, 13 Jul 2020 09:05:59 -0700 Subject: [PATCH 1/2] Handle empty bucket. Allow specifying endpoint url for services like b2 and digitalocean spaces. --- dogsheep_photos/cli.py | 6 +++++- dogsheep_photos/utils.py | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dogsheep_photos/cli.py b/dogsheep_photos/cli.py index 9bf49a3..b973463 100644 --- a/dogsheep_photos/cli.py +++ b/dogsheep_photos/cli.py @@ -44,6 +44,7 @@ def s3_auth(auth): bucket = click.prompt("S3 bucket") access_key_id = click.prompt("Access key ID") secret_access_key = click.prompt("Secret access key") + s3_endpoint = click.prompt("S3 Endpoint (enter black for default)") if pathlib.Path(auth).exists(): auth_data = json.load(open(auth)) else: @@ -51,6 +52,7 @@ def s3_auth(auth): auth_data.update( { "photos_s3_bucket": bucket, + "photos_s3_endpoint": s3_endpoint, "photos_s3_access_key_id": access_key_id, "photos_s3_secret_access_key": secret_access_key, } @@ -86,8 +88,10 @@ def upload(db_path, directories, auth, no_progress, dry_run): "Upload photos from directories to S3" creds = json.load(open(auth)) db = sqlite_utils.Database(db_path) + endpoint_url = creds["photos_s3_endpoint"] if "photos_s3_endpoint" in creds and creds["photos_s3_endpoint"] else None client = boto3.client( "s3", + endpoint_url=endpoint_url, aws_access_key_id=creds["photos_s3_access_key_id"], aws_secret_access_key=creds["photos_s3_secret_access_key"], ) @@ -129,7 +133,7 @@ def upload(db_path, directories, auth, no_progress, dry_run): # Calculate total size first total_size = sum(hash_and_size[p][1] for p in new_paths) click.echo( - "{verb} {num} files, {total_size:.2f} GB".format( + "{verb} {num:,} files, {total_size:.2f} GB".format( verb="Would upload" if dry_run else "Uploading", num=len(new_paths), total_size=total_size / (1024 * 1024 * 1024), diff --git a/dogsheep_photos/utils.py b/dogsheep_photos/utils.py index f9c0d93..2dca54a 100644 --- a/dogsheep_photos/utils.py +++ b/dogsheep_photos/utils.py @@ -43,8 +43,9 @@ def get_all_keys(client, bucket): paginator = client.get_paginator("list_objects_v2") keys = [] for page in paginator.paginate(Bucket=bucket): - for row in page["Contents"]: - keys.append(row["Key"]) + if "Contents" in page: + for row in page["Contents"]: + keys.append(row["Key"]) return keys @@ -118,8 +119,10 @@ def to_uuid(uuid_0, uuid_1): def s3_upload(path, sha256, ext, creds): client = getattr(boto3_local, "client", None) if client is None: + endpoint_url = creds["photos_s3_endpoint"] if "photos_s3_endpoint" in creds and creds["photos_s3_endpoint"] else None client = boto3.client( "s3", + endpoint_url=endpoint_url, aws_access_key_id=creds["photos_s3_access_key_id"], aws_secret_access_key=creds["photos_s3_secret_access_key"], ) From 647d4b42c6f4d1fba4b99f73fe163946cea6ee36 Mon Sep 17 00:00:00 2001 From: Scanner Date: Mon, 13 Jul 2020 09:10:44 -0700 Subject: [PATCH 2/2] Make sure tests pass. --- tests/test_s3_auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_s3_auth.py b/tests/test_s3_auth.py index f53ee1d..5242b26 100644 --- a/tests/test_s3_auth.py +++ b/tests/test_s3_auth.py @@ -6,11 +6,12 @@ def test_s3_auth(): runner = CliRunner() with runner.isolated_filesystem(): - result = runner.invoke(cli, ["s3-auth"], input="bucket\nxxx\nyyy\n") + result = runner.invoke(cli, ["s3-auth"], input="bucket\nxxx\nyyy\nendpoint") assert 0 == result.exit_code data = json.load(open("auth.json")) assert { "photos_s3_bucket": "bucket", "photos_s3_access_key_id": "xxx", "photos_s3_secret_access_key": "yyy", + "photos_s3_endpoint": "endpoint" } == data