From fe36d6c955ef9a0fd16e0ada2b903250d6f45c97 Mon Sep 17 00:00:00 2001 From: Danang Date: Thu, 6 Jun 2024 15:22:07 +0700 Subject: [PATCH 1/4] add transfer_config for s3 storages --- django_project/core/settings/project.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py index 347dae2..e2edbc3 100644 --- a/django_project/core/settings/project.py +++ b/django_project/core/settings/project.py @@ -8,6 +8,7 @@ import os # noqa from django.db import connection +from boto3.s3.transfer import TransferConfig from .contrib import * # noqa from .utils import code_release_version @@ -46,13 +47,20 @@ # s3 # TODO: set CacheControl in object_parameters+endpoint_url +MB = 1024 ** 2 +transfer_config = TransferConfig( + multipart_chunksize=300 * MB, + use_threads=True, + max_concurrency=10 +) STORAGES = { "default": { "BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": { "bucket_name": os.environ.get("AWS_S3_BUCKET_NAME"), "file_overwrite": False, - "max_memory_size": 300 * 1024 * 1024, # 300MB + "max_memory_size": 300 * MB, # 300MB + "transfer_config": transfer_config }, }, "staticfiles": { @@ -63,7 +71,8 @@ "OPTIONS": { "bucket_name": os.environ.get("MINIO_BUCKET_NAME"), "file_overwrite": False, - "max_memory_size": 300 * 1024 * 1024, # 300MB + "max_memory_size": 300 * MB, # 300MB, + "transfer_config": transfer_config }, }, } From 1155f804e21d0b0c542db3b1d9a53299c14bb4a2 Mon Sep 17 00:00:00 2001 From: Danang Date: Thu, 6 Jun 2024 15:57:32 +0700 Subject: [PATCH 2/4] download using boto client --- django_project/core/settings/project.py | 8 ++++---- django_project/cplus_api/models/layer.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py index e2edbc3..e44245d 100644 --- a/django_project/core/settings/project.py +++ b/django_project/core/settings/project.py @@ -48,8 +48,8 @@ # s3 # TODO: set CacheControl in object_parameters+endpoint_url MB = 1024 ** 2 -transfer_config = TransferConfig( - multipart_chunksize=300 * MB, +AWS_TRANSFER_CONFIG = TransferConfig( + multipart_chunksize=512 * MB, use_threads=True, max_concurrency=10 ) @@ -60,7 +60,7 @@ "bucket_name": os.environ.get("AWS_S3_BUCKET_NAME"), "file_overwrite": False, "max_memory_size": 300 * MB, # 300MB - "transfer_config": transfer_config + "transfer_config": AWS_TRANSFER_CONFIG }, }, "staticfiles": { @@ -72,7 +72,7 @@ "bucket_name": os.environ.get("MINIO_BUCKET_NAME"), "file_overwrite": False, "max_memory_size": 300 * MB, # 300MB, - "transfer_config": transfer_config + "transfer_config": AWS_TRANSFER_CONFIG }, }, } diff --git a/django_project/cplus_api/models/layer.py b/django_project/cplus_api/models/layer.py index 1fc1545..36f72f2 100644 --- a/django_project/cplus_api/models/layer.py +++ b/django_project/cplus_api/models/layer.py @@ -127,9 +127,17 @@ def download_to_working_directory(self, base_dir: str): dir_path, os.path.basename(self.file.name) ) - with open(file_path, 'wb+') as destination: - for chunk in self.file.chunks(): - destination.write(chunk) + storage = select_input_layer_storage() + boto3_client = storage.connection.meta.client + boto3_client.download_file( + storage.bucket_name, + self.file.name, + file_path, + Config=settings.AWS_TRANSFER_CONFIG + ) + # with open(file_path, 'wb+') as destination: + # for chunk in self.file.chunks(): + # destination.write(chunk) self.last_used_on = timezone.now() self.save(update_fields=['last_used_on']) if file_path.endswith('.zip'): From e6bb438395024a81e4bc7cf25596c04cac034bf9 Mon Sep 17 00:00:00 2001 From: Danang Date: Thu, 6 Jun 2024 16:07:28 +0700 Subject: [PATCH 3/4] fix test download input layer --- django_project/cplus_api/models/layer.py | 24 ++++++++++--------- .../cplus_api/tests/test_model_layer.py | 4 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/django_project/cplus_api/models/layer.py b/django_project/cplus_api/models/layer.py index 36f72f2..d06a0d9 100644 --- a/django_project/cplus_api/models/layer.py +++ b/django_project/cplus_api/models/layer.py @@ -5,7 +5,7 @@ from django.utils.translation import gettext_lazy as _ from django.conf import settings from django.utils import timezone -from django.core.files.storage import storages +from django.core.files.storage import storages, FileSystemStorage def input_layer_dir_path(instance, filename): @@ -128,16 +128,18 @@ def download_to_working_directory(self, base_dir: str): os.path.basename(self.file.name) ) storage = select_input_layer_storage() - boto3_client = storage.connection.meta.client - boto3_client.download_file( - storage.bucket_name, - self.file.name, - file_path, - Config=settings.AWS_TRANSFER_CONFIG - ) - # with open(file_path, 'wb+') as destination: - # for chunk in self.file.chunks(): - # destination.write(chunk) + if isinstance(storage, FileSystemStorage): + with open(file_path, 'wb+') as destination: + for chunk in self.file.chunks(): + destination.write(chunk) + else: + boto3_client = storage.connection.meta.client + boto3_client.download_file( + storage.bucket_name, + self.file.name, + file_path, + Config=settings.AWS_TRANSFER_CONFIG + ) self.last_used_on = timezone.now() self.save(update_fields=['last_used_on']) if file_path.endswith('.zip'): diff --git a/django_project/cplus_api/tests/test_model_layer.py b/django_project/cplus_api/tests/test_model_layer.py index 0272b0f..299af93 100644 --- a/django_project/cplus_api/tests/test_model_layer.py +++ b/django_project/cplus_api/tests/test_model_layer.py @@ -8,10 +8,10 @@ output_layer_dir_path, InputLayer, ) -from cplus_api.tests.common import BaseInitData +from cplus_api.tests.common import BaseAPIViewTransactionTest -class TestModelLayer(BaseInitData): +class TestModelLayer(BaseAPIViewTransactionTest): def test_input_layer_dir_path(self): # private layer From dbf2f2f127d71f0bf0697ee0ae5387a93ab43867 Mon Sep 17 00:00:00 2001 From: Danang Date: Thu, 6 Jun 2024 16:09:01 +0700 Subject: [PATCH 4/4] fix lint --- django_project/cplus_api/models/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_project/cplus_api/models/layer.py b/django_project/cplus_api/models/layer.py index d06a0d9..00a7a23 100644 --- a/django_project/cplus_api/models/layer.py +++ b/django_project/cplus_api/models/layer.py @@ -139,7 +139,7 @@ def download_to_working_directory(self, base_dir: str): self.file.name, file_path, Config=settings.AWS_TRANSFER_CONFIG - ) + ) self.last_used_on = timezone.now() self.save(update_fields=['last_used_on']) if file_path.endswith('.zip'):