Merge pull request #42 from kartoza/feat-improve-input-layer-prepare

Feat improve input layer prepare

django_project/core/settings/project.py

@@ -8,6 +8,7 @@
 import os  # noqa
 
 from django.db import connection
+from boto3.s3.transfer import TransferConfig
 from .contrib import *  # noqa
 from .utils import code_release_version
 
@@ -46,13 +47,20 @@
 
 # s3
 # TODO: set CacheControl in object_parameters+endpoint_url
+MB = 1024 ** 2
+AWS_TRANSFER_CONFIG = TransferConfig(
+    multipart_chunksize=512 * MB,
+    use_threads=True,
+    max_concurrency=10
+)
 STORAGES = {
     "default": {
         "BACKEND": "storages.backends.s3.S3Storage",
         "OPTIONS": {
           "bucket_name": os.environ.get("AWS_S3_BUCKET_NAME"),
           "file_overwrite": False,
-          "max_memory_size": 300 * 1024 * 1024,  # 300MB
+          "max_memory_size": 300 * MB,  # 300MB
+          "transfer_config": AWS_TRANSFER_CONFIG
         },
     },
     "staticfiles": {
@@ -63,7 +71,8 @@
         "OPTIONS": {
           "bucket_name": os.environ.get("MINIO_BUCKET_NAME"),
           "file_overwrite": False,
-          "max_memory_size": 300 * 1024 * 1024,  # 300MB
+          "max_memory_size": 300 * MB,  # 300MB,
+          "transfer_config": AWS_TRANSFER_CONFIG
         },
     },
 }

django_project/cplus_api/models/layer.py

@@ -5,7 +5,7 @@
 from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.utils import timezone
-from django.core.files.storage import storages
+from django.core.files.storage import storages, FileSystemStorage
 
 
 def input_layer_dir_path(instance, filename):
@@ -127,9 +127,19 @@ def download_to_working_directory(self, base_dir: str):
             dir_path,
             os.path.basename(self.file.name)
         )
-        with open(file_path, 'wb+') as destination:
-            for chunk in self.file.chunks():
-                destination.write(chunk)
+        storage = select_input_layer_storage()
+        if isinstance(storage, FileSystemStorage):
+            with open(file_path, 'wb+') as destination:
+                for chunk in self.file.chunks():
+                    destination.write(chunk)
+        else:
+            boto3_client = storage.connection.meta.client
+            boto3_client.download_file(
+                storage.bucket_name,
+                self.file.name,
+                file_path,
+                Config=settings.AWS_TRANSFER_CONFIG
+            )
         self.last_used_on = timezone.now()
         self.save(update_fields=['last_used_on'])
         if file_path.endswith('.zip'):

django_project/cplus_api/tests/test_model_layer.py

@@ -8,10 +8,10 @@
     output_layer_dir_path,
     InputLayer,
 )
-from cplus_api.tests.common import BaseInitData
+from cplus_api.tests.common import BaseAPIViewTransactionTest
 
 
-class TestModelLayer(BaseInitData):
+class TestModelLayer(BaseAPIViewTransactionTest):
 
     def test_input_layer_dir_path(self):
         # private layer