Handle empty bucket on first upload. Allow specifying the endpoint_url for services other than S3 (like b2 and digitalocean spaces)

dogsheep_photos/cli.py

@@ -44,13 +44,15 @@ def s3_auth(auth):
     bucket = click.prompt("S3 bucket")
     access_key_id = click.prompt("Access key ID")
     secret_access_key = click.prompt("Secret access key")
+    s3_endpoint = click.prompt("S3 Endpoint (enter black for default)")
     if pathlib.Path(auth).exists():
         auth_data = json.load(open(auth))
     else:
         auth_data = {}
     auth_data.update(
         {
             "photos_s3_bucket": bucket,
+            "photos_s3_endpoint": s3_endpoint,
             "photos_s3_access_key_id": access_key_id,
             "photos_s3_secret_access_key": secret_access_key,
         }
@@ -86,8 +88,10 @@ def upload(db_path, directories, auth, no_progress, dry_run):
     "Upload photos from directories to S3"
     creds = json.load(open(auth))
     db = sqlite_utils.Database(db_path)
+    endpoint_url = creds["photos_s3_endpoint"] if "photos_s3_endpoint" in creds and creds["photos_s3_endpoint"] else None
     client = boto3.client(
         "s3",
+        endpoint_url=endpoint_url,
         aws_access_key_id=creds["photos_s3_access_key_id"],
         aws_secret_access_key=creds["photos_s3_secret_access_key"],
     )
@@ -129,7 +133,7 @@ def upload(db_path, directories, auth, no_progress, dry_run):
         # Calculate total size first
         total_size = sum(hash_and_size[p][1] for p in new_paths)
         click.echo(
-            "{verb} {num} files, {total_size:.2f} GB".format(
+            "{verb} {num:,} files, {total_size:.2f} GB".format(
                 verb="Would upload" if dry_run else "Uploading",
                 num=len(new_paths),
                 total_size=total_size / (1024 * 1024 * 1024),

dogsheep_photos/utils.py

@@ -43,8 +43,9 @@ def get_all_keys(client, bucket):
     paginator = client.get_paginator("list_objects_v2")
     keys = []
     for page in paginator.paginate(Bucket=bucket):
-        for row in page["Contents"]:
-            keys.append(row["Key"])
+        if "Contents" in page:
+            for row in page["Contents"]:
+                keys.append(row["Key"])
     return keys
 
 
@@ -118,8 +119,10 @@ def to_uuid(uuid_0, uuid_1):
 def s3_upload(path, sha256, ext, creds):
     client = getattr(boto3_local, "client", None)
     if client is None:
+        endpoint_url = creds["photos_s3_endpoint"] if "photos_s3_endpoint" in creds and creds["photos_s3_endpoint"] else None
         client = boto3.client(
             "s3",
+            endpoint_url=endpoint_url,
             aws_access_key_id=creds["photos_s3_access_key_id"],
             aws_secret_access_key=creds["photos_s3_secret_access_key"],
         )

tests/test_s3_auth.py

@@ -6,11 +6,12 @@
 def test_s3_auth():
     runner = CliRunner()
     with runner.isolated_filesystem():
-        result = runner.invoke(cli, ["s3-auth"], input="bucket\nxxx\nyyy\n")
+        result = runner.invoke(cli, ["s3-auth"], input="bucket\nxxx\nyyy\nendpoint")
         assert 0 == result.exit_code
         data = json.load(open("auth.json"))
         assert {
             "photos_s3_bucket": "bucket",
             "photos_s3_access_key_id": "xxx",
             "photos_s3_secret_access_key": "yyy",
+            "photos_s3_endpoint": "endpoint"
         } == data