[HP-1539] Add AWS functionality to cirrus

.pre-commit-config.yaml

@@ -13,6 +13,6 @@ repos:
     -   id: no-commit-to-branch
         args: [--branch, develop, --branch, master, --branch, main, --pattern, release/.*]
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.3.0
     hooks:
     -   id: black

README.md

@@ -24,6 +24,23 @@ using the library like above.
 
 So... you should at least read how to set up your environment.
 
+For AWS functionality you can use an example like
+
+```
+import boto3
+from gen3cirrus import AwsService
+
+client = boto3.client()
+
+aws = AwsService(client)
+
+object = "test.txt"
+bucket = "testBucket"
+expiration = 3600
+
+url = aws.requestorPaysDownloadPresignedURL( bucket, object, expiration)
+```
+
 ## Setting up Environment for `cirrus`
 `cirrus`'s wispy clouds must dwell in the great blue expanse with other Clouds.
 Thus, you'll need to configure `cirrus` with necessary information about those Clouds
@@ -185,6 +202,14 @@ cirrus_config.update(**settings)
 
 *Still uses Google libraries for auth*
 
+## AWS Specific Implentation Details
+
+### Method for communication with AWS's API(s)
+
+For AWS you must bring your own Boto3 client that you have configured.
+
+You can then setup the AWS service and your client will be passed as an argument to the AWS api.
+
 ## Building the Documentation
 - `pipenv install --dev`
 - `pipenv run python docs/create_docs.py`

gen3cirrus/__init__.py

@@ -1,2 +1,4 @@
 # Expose public API from each cloud provider
 from .google_cloud import GoogleCloudManager
+
+from .aws import AwsCloudManager, AwsService

gen3cirrus/aws/__init__.py

@@ -0,0 +1,2 @@
+from .manager import AwsCloudManager
+from .services import AwsService

gen3cirrus/aws/manager.py

@@ -0,0 +1,7 @@
+class AwsCloudManager:
+    """
+    Manager for AWS cloud clients. Currently this is not in use but has been added in case it is needed in the future
+    """
+
+    def __init__(self):
+        pass

gen3cirrus/aws/services.py

@@ -0,0 +1,27 @@
+"""
+Amazon service for interacting with APIs
+"""
+
+import backoff
+import boto3
+from botocore.exceptions import ClientError
+from gen3cirrus.config import config
+from gen3cirrus.aws.utils import generatePresignedURL, generatePresignedURLRequestorPays
+from cdislogging import get_logger
+
+logger = get_logger(__name__, log_level="info")
+
+
+class AwsService(object):
+    """
+    Generic Amazon servicing using Boto3
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    def downloadPresignedURL(self, bucket, key, expiration):
+        return generatePresignedURL(self.client, bucket, key, expiration)
+
+    def requestorPaysDownloadPresignedURL(self, bucket, key, expiration):
+        return generatePresignedURLRequestorPays(self.client, bucket, key, expiration)

gen3cirrus/aws/utils.py

@@ -0,0 +1,54 @@
+import datetime
+import json
+
+import boto3
+from botocore.exceptions import ClientError
+
+from cdislogging import get_logger
+
+logger = get_logger(__name__, log_level="info")
+
+
+def generatePresignedURL(client, method, bucket_name, object_name, expires):
+    s3_client = client
+
+    if method == "get":
+        m = "get_object"
+    elif method == "put":
+        m = "put_object"
+    else:
+        logger.info(
+            "method for generating presigned url must be 'get' for download or 'put' for upload"
+        )
+        return None
+
+    try:
+        response = s3_client.generate_presigned_url(
+            m, Params={"Bucket": bucket_name, "Key": object_name}, ExpiresIn=expires
+        )
+
+    except ClientError as e:
+        logger.info(e)
+        return None
+
+    return response
+
+
+def generatePresignedURLRequestorPays(client, bucket_name, object_name, expires):
+    s3_client = client
+    try:
+        response = s3_client.generate_presigned_url(
+            "get_object",
+            Params={
+                "Bucket": bucket_name,
+                "Key": object_name,
+                "RequestPayer": "requester",
+            },
+            ExpiresIn=expires,
+        )
+
+    except ClientError as e:
+        logger.info(e)
+        return None
+
+    return response

gen3cirrus/core.py

@@ -4,6 +4,7 @@
 Current Capabilities:
 - Manage Google resources, policies, and access (specific Google APIs
   are abstracted through a Management class that exposes needed behavior)
+- Manage AWS resources amd access S3 APIs
 """
 
 

gen3cirrus/errors.py

@@ -1,5 +1,7 @@
 class CirrusError(Exception):
-    def __init__(self, message="There was an error within the gen3cirrus library.", *args):
+    def __init__(
+        self, message="There was an error within the gen3cirrus library.", *args
+    ):
         super(CirrusError, self).__init__(message)
 
 

pyproject.toml

@@ -16,6 +16,7 @@ google-cloud-storage = "*"
 google-api-python-client = "*"
 google-auth = "*"
 google-auth-httplib2 = "*"
+boto3 = "^1.34.128"
 
 [tool.poetry.group.dev.dependencies]
 mock = "*"

test/test_utils.py

@@ -9,6 +9,12 @@
     get_valid_service_account_id_for_user,
 )
 
+# for testing aws
+import boto3
+import botocore.session
+from botocore.stub import Stubber
+from gen3cirrus.aws.utils import generatePresignedURL, generatePresignedURLRequestorPays
+
 
 @pytest.mark.parametrize(
     "username",
@@ -121,3 +127,27 @@ def test_get_string_to_sign_no_optional_params():
     )
 
     assert result == ("GET\n" "\n" "\n" "1388534400\n" "/bucket/objectname")
+
+
+def test_aws_get_presigned_url():
+    s3 = boto3.client("s3", aws_access_key_id="", aws_secret_access_key="")
+
+    bucket = "test"
+    obj = "test-obj.txt"
+    expires = 3600
+
+    url = generatePresignedURL(s3, "get", bucket, obj, expires)
+
+    assert url != None
+
+
+def test_aws_get_presigned_url_requester_pays():
+    s3 = boto3.client("s3", aws_access_key_id="", aws_secret_access_key="")
+
+    bucket = "test"
+    obj = "test-obj.txt"
+    expires = 3600
+
+    url = generatePresignedURLRequestorPays(s3, bucket, obj, expires)
+
+    assert url != None