diff --git a/bucketListing/Dockerfile b/bucketListing/Dockerfile new file mode 100644 index 0000000..8cfcc19 --- /dev/null +++ b/bucketListing/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11.6-slim + +WORKDIR /usr/src/app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +RUN groupadd -g 999 appuser && \ + useradd -r -u 999 -g appuser appuser +USER appuser + +COPY . . + +ENTRYPOINT [ "python", "./createBucketListing.py"] + diff --git a/bucketListing/createBucketListing.py b/bucketListing/createBucketListing.py new file mode 100644 index 0000000..e2985b1 --- /dev/null +++ b/bucketListing/createBucketListing.py @@ -0,0 +1,28 @@ +from google.cloud import storage +import base64 +import binascii +import argparse + + +def createBucketListing(project, bucket): + + gcsPrefix = "https://storage.neonscience.org/" + gcsBucketName = bucket + storage_client = storage.Client(project) + blobs = storage_client.list_blobs(gcsBucketName) + listing = 'object,last_modified,etag' + '\n' + for blob in blobs: + checksum = binascii.hexlify(base64.urlsafe_b64decode(blob.md5_hash)).decode() + listing += gcsPrefix + gcsBucketName + "/" + blob.name + ',' + blob.updated.strftime("%Y-%m-%d %H:%M:%S") + ',' + checksum + '\n' + + gcsBucket = storage_client.bucket(gcsBucketName) + listingBlob = gcsBucket.blob("listing.csv") + listingBlob.upload_from_string(listing) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--project') + parser.add_argument('--bucket') + args=parser.parse_args() + createBucketListing(args.project, args.bucket) \ No newline at end of file diff --git a/bucketListing/requirements.txt b/bucketListing/requirements.txt new file mode 100644 index 0000000..c2393a2 --- /dev/null +++ b/bucketListing/requirements.txt @@ -0,0 +1,17 @@ +cachetools==5.3.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +google-api-core==2.14.0 +google-auth==2.23.4 +google-cloud-core==2.3.3 +google-cloud-storage==2.13.0 +google-crc32c==1.5.0 +google-resumable-media==2.6.0 +googleapis-common-protos==1.61.0 +idna==3.6 +protobuf==4.25.1 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 +requests==2.31.0 +rsa==4.9 +urllib3==2.1.0