[HP-1539] Add AWS functionality to cirrus

.pre-commit-config.yaml

@@ -13,6 +13,6 @@ repos:
     -   id: no-commit-to-branch
         args: [--branch, develop, --branch, master, --branch, main, --pattern, release/.*]
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.3.0
     hooks:
     -   id: black

README.md

@@ -24,6 +24,23 @@ using the library like above.
 
 So... you should at least read how to set up your environment.
 
+For AWS functionality you can use an example like
+
+```
+import boto3
+from gen3cirrus import AwsService
+
+client = boto3.client()
+
+aws = AwsService(client)
+
+object = "test.txt"
+bucket = "testBucket"
+expiration = 3600
+
+url = aws.requester_pays_download_presigned_url(bucket, object, expiration)
+```
+
 ## Setting up Environment for `cirrus`
 `cirrus`'s wispy clouds must dwell in the great blue expanse with other Clouds.
 Thus, you'll need to configure `cirrus` with necessary information about those Clouds
@@ -32,7 +49,7 @@ before being able to bask in its beauty.
 You *should* only have to do this once so don't freak out.
 
 By default, all the configurations needed by `cirrus` are assumed to be environmental
-variables. You can also provide the configuration programatically in Python (instructions are later in the README).
+variables. You can also provide the configuration programmatically in Python (instructions are later in the README).
 
 **Note:** This guide should cover necessary configuration,
 but in the effort of not having to maintain everything in two places,
@@ -106,7 +123,7 @@ a few guides on settings that up, as it requires you to enable access to the
 Cloud Identity/GSuite API.
 
 Follow directions [here](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority)
-to deletgate domain-wide authority for your service account that you're using
+to delegate domain-wide authority for your service account that you're using
 for `GOOGLE_APPLICATION_CREDENTIALS`.
 
 For the API scopes, authorize these:
@@ -141,7 +158,7 @@ GOOGLE_API_KEY="abcdefghijklmnopqrstuvwxyz"
 ```
 
 ### Setting Configuration Programatically
-`cirrus`, by default, reads in necessary configurations from environmental variables. You can, however, provide all these config vars programatically by calling the `update` function on the config object in `cirrus` and passing in a dictionary.
+`cirrus`, by default, reads in necessary configurations from environmental variables. You can, however, provide all these config vars programmatically by calling the `update` function on the config object in `cirrus` and passing in a dictionary.
 
 For example:
 ```
@@ -185,6 +202,14 @@ cirrus_config.update(**settings)
 
 *Still uses Google libraries for auth*
 
+## AWS Specific Implentation Details
+
+### Method for communication with AWS's API(s)
+
+For AWS you must bring your own Boto3 client that you have configured.
+
+You can then setup the AWS service and your client will be passed as an argument to the AWS API.
+
 ## Building the Documentation
 - `pipenv install --dev`
 - `pipenv run python docs/create_docs.py`

gen3cirrus/aws/__init__.py

@@ -0,0 +1 @@
+from .services import AwsService

gen3cirrus/aws/services.py

@@ -0,0 +1,57 @@
+"""
+Amazon service for interacting with APIs
+"""
+
+
+from gen3cirrus.aws.utils import (
+    generate_presigned_url,
+    generate_presigned_url_requester_pays,
+    generate_multipart_upload_url,
+)
+
+from cdislogging import get_logger
+
+logger = get_logger(__name__, log_level="info")
+
+
+class AwsService(object):
+    """
+    Generic Amazon services using Boto3
+    """
+
+    def __init__(self, boto3_client):
+        self.client = boto3_client
+
+    def download_presigned_url(self, bucket, key, expiration, additional_info=None):
+        """
+        Wrapper function for generating a presigned URL for downloading an object
+        """
+        return generate_presigned_url(
+            self.client, "get", bucket, key, expiration, additional_info
+        )
+
+    def upload_presigned_url(self, bucket, key, expiration, additional_info=None):
+        """
+        Wrapper function for generating a presigned URL for uploading an object
+        """
+        return generate_presigned_url(
+            self.client, "put", bucket, key, expiration, additional_info
+        )
+
+    def multipart_upload_presigned_url(self, bucket, key, expiration, upload_id, part):
+        """
+        Wrapper function for generating a presigned URL for uploading an object using multipart upload
+        """
+        return generate_multipart_upload_url(
+            self.client, bucket, key, expiration, upload_id, part
+        )
+
+    def requester_pays_download_presigned_url(
+        self, bucket, key, expiration, additional_info=None
+    ):
+        """
+        Wrapper function for generating a presigned URL for downloading an object from a requester pays bucket
+        """
+        return generate_presigned_url_requester_pays(
+            self.client, bucket, key, expiration, additional_info
+        )

gen3cirrus/aws/utils.py

@@ -0,0 +1,128 @@
+from botocore.exceptions import ClientError
+
+from cdislogging import get_logger
+
+logger = get_logger(__name__, log_level="info")
+
+
+def generate_presigned_url(
+    client, method, bucket_name, object_name, expires, additional_info=None
+):
+    """
+    Function for generating a presigned URL for upload or download
+
+    Args:
+        client (S3.Client): boto3 S3 client
+        method (string): ["get", "put"] "get" for download and "put" for upload
+        bucket_name (string): s3 bucket name
+        object_name (string): s3 bucket object key
+        expires (int): time for presigned URL to exist (in seconds)
+        additional_info (dict): dict of additional parameters to pass to s3 for signing
+    """
+
+    params = {}
+    params["Bucket"] = bucket_name
+    params["Key"] = object_name
+
+    additional_info = additional_info or {}
+    for key in additional_info:
+        params[key] = additional_info[key]
+
+    s3_client = client
+
+    if method == "get":
+        client_method = "get_object"
+    elif method == "put":
+        client_method = "put_object"
+    else:
+        logger.error(
+            "method for generating presigned URL must be 'get' for download or 'put' for upload"
+        )
+        return None
+
+    try:
+        response = s3_client.generate_presigned_url(
+            client_method,
+            Params=params,
+            ExpiresIn=expires,
+        )
+
+    except ClientError as e:
+        logger.error(e)
+        return None
+
+    return response
+
+
+def generate_multipart_upload_url(
+    client, bucket_name, object_name, expires, upload_id, part_no
+):
+    """
+    Function for generating a presigned URL only for one part of multipart upload
+
+    Args:
+        client (S3.Client): boto3 S3 client
+        method (string): ["get", "put"] "get" for download and "put" for upload
+        bucket_name (string): s3 bucket name
+        object_name (string): s3 bucket object key
+        expires (int): time for presigned URL to exist (in seconds)
+        upload_id (string): ID for upload to s3
+        part_no (int): part number of multipart upload
+    """
+    s3_client = client
+    try:
+        response = s3_client.generate_presigned_url(
+            ClientMethod="upload_part",
+            Params={
+                "Bucket": bucket_name,
+                "Key": object_name,
+                "UploadId": upload_id,
+                "PartNumber": part_no,
+            },
+            ExpiresIn=expires,
+        )
+
+    except ClientError as e:
+        logger.error(e)
+        return None
+
+    return response
+
+
+def generate_presigned_url_requester_pays(
+    client, bucket_name, object_name, expires, additional_info=None
+):
+    """
+    Function for generating a presigned URL only for requester pays buckets
+
+    Args:
+        client (S3.Client): boto3 S3 client
+        method (string): ["get", "put"] "get" for download and "put" for upload
+        bucket_name (string): s3 bucket name
+        object_name (string): s3 bucket object key
+        expires (int): time for presigned URL to exist (in seconds)
+        additional_info (dict): dict of additional parameters to pass to s3 for signing
+    """
+    params = {}
+    params["Bucket"] = bucket_name
+    params["Key"] = object_name
+    params["RequestPayer"] = "requester"
+
+    additional_info = additional_info or {}
+    for key in additional_info:
+        params[key] = additional_info[key]
+
+    s3_client = client
+
+    try:
+        response = s3_client.generate_presigned_url(
+            "get_object",
+            Params=params,
+            ExpiresIn=expires,
+        )
+
+    except ClientError as e:
+        logger.error(e)
+        return None
+
+    return response

gen3cirrus/core.py

@@ -4,6 +4,7 @@
 Current Capabilities:
 - Manage Google resources, policies, and access (specific Google APIs
   are abstracted through a Management class that exposes needed behavior)
+- Manage AWS resources amd access S3 APIs
 """
 
 

gen3cirrus/errors.py

@@ -1,5 +1,7 @@
 class CirrusError(Exception):
-    def __init__(self, message="There was an error within the gen3cirrus library.", *args):
+    def __init__(
+        self, message="There was an error within the gen3cirrus library.", *args
+    ):
         super(CirrusError, self).__init__(message)