Merge pull request #217 from FAIRDataPipeline/update/object-storage

Update/object storage

.github/workflows/fair-data-registry.yaml

@@ -22,18 +22,12 @@ jobs:
               else
                   brew install graphviz
               fi
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
       - name: Run Tests
         run: |
-          poetry install 
-          source $VENV
+          python -m pip install -r local-requirements.txt
           DJANGO_SETTINGS_MODULE=drams.test-settings coverage run --omit=drams,scripts,tools manage.py test 
       - name: Generate XML
-        run: poetry run coverage xml
+        run: coverage xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

data_management/object_storage.py

@@ -1,18 +1,26 @@
-from hashlib import sha1
-import hmac
-import time
+import boto3
 
-from . import settings
+from django.conf import settings
 
-def create_url(path, method, filename=None):
-    expiry_time = int(time.time() + int(settings.CONFIG.get('storage', 'duration')))
-    path = settings.CONFIG.get('storage', 'bucket') + '/' + path
-    if method == 'GET':
-        hmac_body = '%s\n%s\n%s' % ('GET', expiry_time, path)
-    elif method == 'PUT':
-        hmac_body = '%s\n%s\n%s' % ('PUT', expiry_time, path)
-    sig = hmac.new(settings.CONFIG.get('storage', 'key').encode('utf-8'), hmac_body.encode('utf-8'), sha1).hexdigest()
-    url = '%s%s?temp_url_sig=%s&temp_url_expires=%d' % (settings.CONFIG.get('storage', 'url'), path, sig, expiry_time)
-    if filename:
-        url = '%s&filename=%s' % (url, filename)
-    return url
+def create_url(name, method, filename = None):
+    bucket = settings.BUCKETS['default']
+    session = boto3.session.Session()
+    s3_client = session.client(
+        service_name= 's3',
+        aws_access_key_id= bucket['access_key'],
+        aws_secret_access_key= bucket['secret_key'],
+        endpoint_url= bucket['url'],
+    )
+    if method == "GET":
+        response = s3_client.generate_presigned_url('get_object',
+                                                        Params={'Bucket': bucket['bucket_name'],
+                                                                'Key': name,
+                                                                'ResponseContentDisposition': f'attachment; filename = {filename}',},
+                                                        ExpiresIn=bucket['duration'])
+    else:
+        response = s3_client.generate_presigned_url('put_object',
+                                                        Params={'Bucket': bucket['bucket_name'],
+                                                                'Key': name},
+                                                        ExpiresIn=bucket['duration'])
+
+    return response

data_management/settings.py

@@ -1,13 +1,6 @@
+from django.conf import settings
 
-import configparser
-import os
-import sys
-
-from django.conf import settings as conf_settings
-
-CONFIG = configparser.ConfigParser()
-if os.path.exists(conf_settings.CONFIG_LOCATION):
-    CONFIG.read(conf_settings.CONFIG_LOCATION)
-    REMOTE_REGISTRY = True
+if settings.AUTHORISED_USER_FILE:
+    REMOTE_REGISTRY = settings.REMOTE
 else:
-    REMOTE_REGISTRY = False
+    REMOTE_REGISTRY = False

drams/base_settings.py

@@ -16,6 +16,8 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = False
 
+REMOTE = False
+
 ALLOWED_HOSTS = ['127.0.0.1', 'localhost']
 
 # The URL of the central public registry

drams/local-settings.py

@@ -7,4 +7,10 @@
 ALLOWED_HOSTS.extend(
     filter(None, os.environ.get("FAIR_ALLOWED_HOSTS", "").split(","))
 )
-DOMAIN_URL = "http://127.0.0.1:8000/"
+DOMAIN_URL = "http://127.0.0.1:8000/"
+
+DEBUG = True
+CONFIG_LOCATION = "example_config.ini"
+SOCIAL_AUTH_GITHUB_KEY = 'a1b2c3d4'
+SOCIAL_AUTH_GITHUB_SECRET = 'e5f6g7h8i9'
+AUTH_METHOD = "GitHub"

drams/settings.py

@@ -5,6 +5,8 @@
 
 ALLOWED_HOSTS = ['data.fairdatapipeline.org', '127.0.0.1', 'localhost']
 
+REMOTE = True
+
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
@@ -26,5 +28,13 @@
     }
 }
 
-CONFIG_LOCATION = os.path.join(os.path.expanduser('~'), 'config.ini')
+BUCKETS = {
+    'default': {
+       'url' : '#',
+       'bucket_name': '#',
+       'access_key': '#',
+       'secret_key': '#',
+       'duration': '600'
+    }
+}
 CACHE_DURATION = 300

drams/test-remote-settings.py

@@ -7,6 +7,17 @@
 ALLOWED_HOSTS.extend(
     filter(None, os.environ.get("FAIR_ALLOWED_HOSTS", "").split(","))
 )
-CONFIG_LOCATION = "example_config.ini"
+
+BUCKETS = {
+    'default': {
+       'url' : 'http://127.0.0.1:4566/',
+       'bucket_name': 'fair',
+       'access_key': 'AccessKey',
+       'secret_key': 'SecretKey',
+       'duration': '60'
+    }
+}
+
+REMOTE = True
 DOMAIN_URL = 'http://127.0.0.1:8001/'
 DEBUG = True

local-requirements.txt

@@ -38,3 +38,4 @@ setuptools>54.2
 whitenoise==5.2.0
 coverage
 rocrate==0.7.0
+boto3>=1.24