feat: add Google Cloud Storage support (GCS)

openassessment/fileupload/backends/__init__.py

@@ -3,7 +3,7 @@
 
 from django.conf import settings
 
-from . import django_storage, filesystem, s3, swift
+from . import django_storage, filesystem, gcs, s3, swift
 
 
 def get_backend():
@@ -18,6 +18,8 @@ def get_backend():
         return filesystem.Backend()
     elif backend_setting == "swift":
         return swift.Backend()
+    elif backend_setting == "gcs":
+        return gcs.Backend()
     elif backend_setting == "django":
         return django_storage.Backend()
     else:

openassessment/fileupload/backends/gcs.py

@@ -0,0 +1,73 @@
+"""GCS Bucket File Upload Backend."""
+import functools
+import logging
+
+from ..exceptions import FileUploadInternalError
+from .base import BaseBackend
+
+log = logging.getLogger("openassessment.fileupload.api")  # pylint: disable=invalid-name
+
+
+def catch_broad_exception(method):
+    """Decorator to catch broad exceptions, log them, and raise a FileUploadInternalError."""
+    @functools.wraps(method)
+    def wrapper(*args, **kwargs):
+        try:
+            return method(*args, **kwargs)
+        except Exception as ex:  # pylint: disable=broad-except
+            log.exception(
+                f"Internal exception occurred while executing ora2 file-upload backend gcs.{method.__name__}: {str(ex)}"
+            )
+            raise FileUploadInternalError(ex) from ex
+    return wrapper
+
+
+class Backend(BaseBackend):
+    """ GCS Bucket File Upload Backend. """
+
+    @catch_broad_exception
+    def get_upload_url(self, key, content_type):
+        """Get a signed URL for uploading a file to GCS"""
+        bucket_name, key_name = self._retrieve_parameters(key)
+        blob = get_blob_object(bucket_name, key_name)
+
+        return blob.generate_signed_url(
+            version="v4",
+            expiration=self.UPLOAD_URL_TIMEOUT,
+            method="PUT",
+            content_type=content_type,
+        )
+
+    @catch_broad_exception
+    def get_download_url(self, key):
+        """Get a signed URL for downloading a file from GCS"""
+        bucket_name, key_name = self._retrieve_parameters(key)
+        blob = get_blob_object(bucket_name, key_name)
+        if not blob.exists():
+            return ""
+
+        return blob.generate_signed_url(
+            version="v4",
+            expiration=self.DOWNLOAD_URL_TIMEOUT,
+            method="GET",
+        )
+
+    @catch_broad_exception
+    def remove_file(self, key):
+        """Remove a file from GCS"""
+        bucket_name, key_name = self._retrieve_parameters(key)
+        blob = get_blob_object(bucket_name, key_name)
+        if blob.exists():
+            blob.delete()
+            return True
+
+        return False
+
+
+def get_blob_object(bucket_name, key_name):
+    """Get a blob object from GCS"""
+    # By default; avoid the need of google-cloud-storage library. It will be only needed if gcs backend is used.
+    from google.cloud import storage  # pylint: disable=import-outside-toplevel
+
+    client = storage.Client()
+    return client.bucket(bucket_name).blob(key_name)

openassessment/fileupload/tests/test_api.py

@@ -19,6 +19,7 @@
 from openassessment.fileupload import api, exceptions, urls
 from openassessment.fileupload import views_filesystem as views
 from openassessment.fileupload.backends.base import Settings as FileUploadSettings
+from openassessment.fileupload.backends.gcs import get_blob_object
 from openassessment.fileupload.backends.filesystem import (
     get_cache as get_filesystem_cache,
 )
@@ -499,3 +500,140 @@ def test_remove(self, key):
         # File no longer exists
         download_url = self.backend.get_download_url(self.key)
         self.assertIsNone(download_url)
+
+
+@override_settings(
+    ORA2_FILEUPLOAD_BACKEND="gcs",
+    DEFAULT_FILE_STORAGE="storages.backends.gcloud.GoogleCloudStorage",
+    FILE_UPLOAD_STORAGE_PREFIX="submissions_on_gcp",
+    FILE_UPLOAD_STORAGE_BUCKET_NAME="testbucket",
+    LMS_ROOT_URL="http://foobar.example.com",
+)
+@ddt.ddt
+class TestFileUploadServiceWithGoogleStorageBackend(TestCase):
+    """
+    Test open assessment file upload using GCS storage backend.
+    """
+    class MockBlob:
+        def __init__(self):
+            self.exist_testing_flag = False
+            self.delete_called_during_test = False
+
+        def generate_signed_url(self, **kwargs):
+            return "http://foobar.example.com/submissions_on_gcs/signed_url_called.txt"
+
+        def delete(self):
+            self.delete_called_during_test = True
+
+        def exists(self):
+            return self.exist_testing_flag
+
+    def setUp(self):
+        super().setUp()
+        self.backend = api.backends.get_backend()
+        # self.username = "test_user"
+        # self.password = "password"
+        # self.user = get_user_model().objects.create_user(
+        #     username=self.username, password=self.password
+        # )
+        #
+        # self.content = tempfile.TemporaryFile()
+        # self.content.write(b"foobar content")
+        # self.content.seek(0)
+
+        # self.key = "myfile.txt"
+        # self.content_type = "text/plain"
+        # self.base_url = "http://foobar.example.com"
+        self.patchers = {
+            "get_blob_object": patch(
+                "openassessment.fileupload.backends.gcs.get_blob_object",
+                return_value=self.MockBlob(),
+            ),
+            "log": patch("openassessment.fileupload.backends.gcs.log.exception"),
+        }
+
+        self.mock_get_blob_object = self.patchers["get_blob_object"].start()
+        self.mock_log = self.patchers["log"].start()
+
+    def tearDown(self):
+        """
+        Stop the patcher.
+        """
+        for patcher in self.patchers.values():
+            patcher.stop()
+
+    def test_get_backend(self):
+        """
+        Ensure the django storage backend is returned when ORA2_FILEUPLOAD_BACKEND="gcs".
+        """
+        self.assertTrue(isinstance(self.backend, api.backends.gcs.Backend))
+
+    @ddt.data(
+        ("get_upload_url", {'key': 'whatever', 'content_type': 'whatever'}),
+        ("get_download_url", {'key': 'whatever'}),
+        ("remove_file", {'key': 'whatever'}),
+    )
+    @ddt.unpack
+    def test_errors_raise_file_upload_internal_error(self, method_name, kwargs):
+        """
+        Ensure that exceptions are caught and raised as FileUploadInternalError.
+        """
+        self.mock_get_blob_object.side_effect = Exception("Some error!")
+        with raises(exceptions.FileUploadInternalError):
+            getattr(self.backend, method_name)(**kwargs)
+        self.mock_log.assert_called_once_with(
+            f"Internal exception occurred while executing ora2 file-upload backend gcs.{method_name}: Some error!"
+        )
+
+    def test_get_upload_url(self):
+        """
+        Verify the upload URL.
+        """
+        url = self.backend.get_upload_url("foo", "_text")
+        self.mock_get_blob_object.assert_called_once_with("testbucket", "submissions_on_gcp/foo")
+        self.assertEqual(url, "http://foobar.example.com/submissions_on_gcs/signed_url_called.txt")
+
+    def test_get_download_url(self):
+        """
+        Verify the download URL.
+        """
+        url = self.backend.get_download_url("foo")
+        self.mock_get_blob_object.assert_called_once_with("testbucket", "submissions_on_gcp/foo")
+        self.assertEqual(url, "")
+
+        self.mock_get_blob_object.return_value.exist_testing_flag = True
+        url = self.backend.get_download_url("foo")
+        self.assertEqual(url, "http://foobar.example.com/submissions_on_gcs/signed_url_called.txt")
+
+    def test_remove_file(self):
+        """
+        Verify the remove file method.
+        """
+        self.assertFalse(self.mock_get_blob_object.return_value.delete_called_during_test)
+        result = self.backend.remove_file("foo")
+        self.mock_get_blob_object.assert_called_once_with("testbucket", "submissions_on_gcp/foo")
+        self.assertFalse(result)
+
+        self.mock_get_blob_object.return_value.exist_testing_flag = True
+        self.assertFalse(self.mock_get_blob_object.return_value.delete_called_during_test)
+        result = self.backend.remove_file("foo")
+        self.assertTrue(result)
+        self.assertTrue(self.mock_get_blob_object.return_value.delete_called_during_test)
+
+    def test_get_blob_object(self):
+        """
+        Verify the get_blob_object method.
+        """
+        with patch("google.cloud.storage") as mock_storage:
+            blob = Mock(id="the_test_blob")
+
+            bucket = Mock()
+            bucket.blob.return_value = blob
+
+            client = Mock()
+            client.bucket.return_value = bucket
+
+            mock_storage.Client.return_value = client
+
+            result = get_blob_object("testbucket", "submissions_on_gcp/foo")
+            self.assertEqual(result.id, "the_test_blob")