Merge pull request #212 from FAIRDataPipeline/feature/repo_authentica…

…tion

Feature/repo authentication

custom_user/models.py

@@ -1,15 +1,57 @@
 from django.contrib.auth.models import AbstractUser, AbstractBaseUser
-import mysql.connector as mariadb
+from django.core.exceptions import ValidationError
+from django.conf import settings
+import yaml
 
 from .managers import CustomUserManager
 
+def _get_users():
+    _authorised_users_file = settings.AUTHORISED_USER_FILE
+    if _authorised_users_file:
+        try:
+            _authorised_users = yaml.safe_load(open(_authorised_users_file))
+            if "users" in _authorised_users:
+                return _authorised_users["users"]
+        except Exception as e:
+            return []
+    return []
+
+def _get_user(username):
+    _users = _get_users()
+    for _user in _users:
+        if "username" in _user:
+            if username == _user["username"]:
+                return _user
+    return None
+
+def _is_valid_user(username):
+    if _get_user(username):
+        return True
+    return False
+
+def _get(key, username):
+    _user = _get_user(username)
+    if _user:
+        if key in _user:
+            return _user[key]
+    return None
+
+def _get_full_name(username):
+    if _get("fullname", username):
+        return _get("fullname", username)
+    return 'User Not Found'
 
-def _get_db_connection():
-    connection = mariadb.connect(
-        option_files='/home/ubuntu/.mysql/people.cnf',
-        database='people'
-    )
-    return connection
+def _get_email(username):
+    _user = _get_user(username)
+    if _user:
+        if "email" in _user:
+            return _user["email"]
+    return f'{username}@users.noreply.github.com'
+
+def _get_orgs(username):
+    if _get("orgs", username):
+        return _get("orgs", username)
+    return []
 
 
 class User(AbstractUser):
@@ -21,51 +63,25 @@ class User(AbstractUser):
     REQUIRED_FIELDS = []
 
     def full_name(self):
-        conn = None
-        try:
-            conn = _get_db_connection()
-            cursor = conn.cursor()
-            cursor.execute('SELECT full_name FROM user WHERE name = %s', (self.username,))
-            row = cursor.fetchone()
-            return row[0]
-        except (ValueError, TypeError):
-            return 'User Not Found'
-        finally:
-            if conn is not None: conn.close()
+        return _get_full_name(self.username)
 
     def email(self):
-        conn = None
-        try:
-            conn = _get_db_connection()
-            cursor = conn.cursor()
-            cursor.execute('SELECT email FROM user WHERE name = %s', (self.username,))
-            row = cursor.fetchone()
-            return row[0]
-        except (ValueError, TypeError):
-            return 'User Not Found'
-        finally:
-            if conn is not None: conn.close()
+        return _get_email(self.username)
 
     def orgs(self):
-        sql = '''
-        SELECT org.name FROM user, org, user_orgs 
-        WHERE user.name = %s
-          AND user_orgs.user_id = user.id
-          AND user_orgs.org_id = org.id
-        '''
-        conn = None
-        try:
-            conn = _get_db_connection()
-            cursor = conn.cursor()
-            cursor.execute(sql, (self.username,))
-            rows = cursor.fetchmany()
-            return [row[0] for row in rows]
-        except (ValueError, TypeError):
-            return []
-        finally:
-            if conn is not None:
-                conn.close()
+        return _get_orgs(self.username)
 
     def clean(self):
         # Skip the AbstractUser.clean as this tries to set self.email
+        if not self.is_superuser:
+            if _get_users():
+                if not _is_valid_user(self.username):
+                    raise ValidationError(
+                    {'username': "Username is not in allowed users"})
         AbstractBaseUser.clean(self)
+
+    def save(self, *args, **kwargs):
+        # This overriding of the save method is necessary because Django by default does not call the
+        # full_clean() method and there is where the clean() method is called
+        self.clean()
+        return super().save(*args, **kwargs)

data_management/prov.py

@@ -741,7 +741,7 @@ def highlight_issues(dot):
                 for cell in row:
                     if (
                         "href" in cell.attrib
-                        and cell.attrib["href"] == "https://data.scrc.uk/vocab/#issue"
+                        and cell.attrib["href"] == "https://data.fairdatapipeline.org/vocab/#issue"
                     ):
                         cell.attrib["bgcolor"] = "red"
             new_label = xml.etree.ElementTree.tostring(table, encoding="unicode")

data_management/tests/init_prov_db.py

@@ -38,7 +38,7 @@ def init_db():
 
     sr_textfiles = StorageRoot.objects.create(
         updated_by=user,
-        root="https://data.scrc.uk/api/text_file/",
+        root="https://data.fairdatapipeline.org/api/text_file/",
     )
 
     sr_example = StorageRoot.objects.create(

data_management/tests/initdb.py

@@ -40,7 +40,7 @@ def init_db(test=True):
 
     sr_textfiles = StorageRoot.objects.create(
         updated_by=user,
-        root='https://data.scrc.uk/api/text_file/',
+        root='https://data.fairdatapipeline.org/api/text_file/',
     )
 
     sr_scot = StorageRoot.objects.create(

data_management/tests/test_api.py

@@ -887,7 +887,7 @@ def test_get_json(self):
 
         expected_result = {
             self.RDF_TYPE: {"$": self.DCAT_DATASET, "type": self.PROV_QUALIFIED_NAME},
-            self.PROV_AT_LOCATION: "https://data.scrc.uk/api/text_file/input/1",
+            self.PROV_AT_LOCATION: "https://data.fairdatapipeline.org/api/text_file/input/1",
             self.DCTERMS_DESCRIPTION: "input 1 object",
             self.FAIR_NAMESPACE: "prov",
             self.DCTERMS_TITLE: "this/is/cr/test/input/1",
@@ -899,7 +899,7 @@ def test_get_json(self):
 
         expected_result = {
             self.RDF_TYPE: {"$": self.DCAT_DATASET, "type": self.PROV_QUALIFIED_NAME},
-            self.PROV_AT_LOCATION: "https://data.scrc.uk/api/text_file/output/1",
+            self.PROV_AT_LOCATION: "https://data.fairdatapipeline.org/api/text_file/output/1",
             self.DCTERMS_DESCRIPTION: "output 1 object",
             self.FAIR_NAMESPACE: "prov",
             self.DCTERMS_TITLE: "this/is/cr/test/output/1",
@@ -911,7 +911,7 @@ def test_get_json(self):
 
         expected_result = {
             self.RDF_TYPE: {"$": self.DCAT_DATASET, "type": self.PROV_QUALIFIED_NAME},
-            self.PROV_AT_LOCATION: "https://data.scrc.uk/api/text_file/input/2",
+            self.PROV_AT_LOCATION: "https://data.fairdatapipeline.org/api/text_file/input/2",
             self.DCTERMS_DESCRIPTION: "input 2 object",
             self.DCTERMS_FORMAT: self.TEXT_FILE,
             self.FAIR_NAMESPACE: "prov",
@@ -924,7 +924,7 @@ def test_get_json(self):
 
         expected_result = {
             self.RDF_TYPE: {"$": self.DCAT_DATASET, "type": self.PROV_QUALIFIED_NAME},
-            self.PROV_AT_LOCATION: "https://data.scrc.uk/api/text_file/input/3",
+            self.PROV_AT_LOCATION: "https://data.fairdatapipeline.org/api/text_file/input/3",
             self.DCTERMS_DESCRIPTION: "input 3 object",
             self.DCTERMS_FORMAT: self.TEXT_FILE,
             self.FAIR_NAMESPACE: "prov",
@@ -985,15 +985,15 @@ def test_get_json(self):
         self.assertEqual(prov_out, expected_result)
 
         expected_result = {
-            self.PROV_AT_LOCATION: "https://data.scrc.uk/api/text_file/15/?format=text"
+            self.PROV_AT_LOCATION: "https://data.fairdatapipeline.org/api/text_file/15/?format=text"
         }
         prov_out = results["entity"][f"{self.LREG_OBJECT}3"]
         del prov_out[self.DCTERMS_MODIFIED]
         self.assertEqual(prov_out, expected_result)
 
         expected_result = {
             self.DCTERMS_FORMAT: self.TEXT_FILE,
-            self.PROV_AT_LOCATION: "https://data.scrc.uk/api/text_file/16/?format=text",
+            self.PROV_AT_LOCATION: "https://data.fairdatapipeline.org/api/text_file/16/?format=text",
             self.RDF_TYPE: {
                 "$": "dcmitype:Software",
                 "type": self.PROV_QUALIFIED_NAME,
@@ -1200,14 +1200,14 @@ def test_get_provn(self):
         result = result_bits[0] + result_end
         expected_result = """document
   prefix lreg <http://localhost/>
-  prefix fair <https://data.scrc.uk/vocab/#>
+  prefix fair <https://data.fairdatapipeline.org/vocab/#>
   prefix dcat <http://www.w3.org/ns/dcat#>
   prefix dcmitype <http://purl.org/dc/dcmitype/>
   prefix dcterms <http://purl.org/dc/terms/>
   prefix foaf <http://xmlns.com/foaf/spec/#>
   prefix rdf <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
   
-  entity(lreg:api/data_product/1, [rdf:type='dcat:Dataset', prov:atLocation="https://data.scrc.uk/api/text_file/input/1", dcterms:description="input 1 object", fair:namespace="prov", dcterms:title="this/is/cr/test/input/1", dcat:hasVersion="0.2.0"])
+  entity(lreg:api/data_product/1, [rdf:type='dcat:Dataset', prov:atLocation="https://data.fairdatapipeline.org/api/text_file/input/1", dcterms:description="input 1 object", fair:namespace="prov", dcterms:title="this/is/cr/test/input/1", dcat:hasVersion="0.2.0"])
   agent(lreg:api/author/1, [rdf:type='prov:Person', foaf:name="Ivana Valenti"])
   wasAttributedTo(lreg:api/data_product/1, lreg:api/author/1, [prov:role='dcterms:creator'])
   entity(lreg:api/external_object/1, [rdf:type='dcat:Dataset', dcterms:title="this is cr test input 1", dcterms:issued="2020-07-10T18:38:00+00:00" %% xsd:dateTime, dcat:hasVersion="0.2.0", fair:alternate_identifier="this_is_cr_test_input_1", fair:alternate_identifier_type="text", dcterms:description="this is code run test input 1", prov:atLocation="https://example.org/file_strore/1.txt"])

docs/api_guide.md

@@ -58,7 +58,7 @@ requests.
 
 ```python
 import requests
-r = requests.get('https://data.scrc.uk/api/object/')
+r = requests.get('https://data.fairdatapipeline.org/api/object/')
 
 # 200 is successful request
 assert(r.status_code == 200)
@@ -85,7 +85,7 @@ print(objects[0]["description"])
 
 ```python
 import requests
-r = requests.get('https://data.scrc.uk/api/object/31946/')
+r = requests.get('https://data.fairdatapipeline.org/api/object/31946/')
 
 # 200 is successful request
 assert(r.status_code == 200)
@@ -109,7 +109,7 @@ import requests
 headers = {
     'Authorization': 'token <TOKEN>'
 }
-r = requests.options('https://data.scrc.uk/api/object/31946/', headers=headers)
+r = requests.options('https://data.fairdatapipeline.org/api/object/31946/', headers=headers)
 
 # 200 is successful request
 assert(r.status_code == 200)
@@ -145,9 +145,9 @@ headers = {
 data = {
     "path": "path/to/file",
     "hash": "dae8dd8b2837e2f33979da109109244f0f9e273f",
-    "storage_root": "https://data.scrc.uk/api/storage_root/1/"
+    "storage_root": "https://data.fairdatapipeline.org/api/storage_root/1/"
 }
-r = requests.put('https://data.scrc.uk/api/object/31946/', data, headers=headers)
+r = requests.put('https://data.fairdatapipeline.org/api/object/31946/', data, headers=headers)
 
 # 201 is successful create
 assert(r.status_code == 201)

docs/authentication.md

@@ -5,16 +5,16 @@
 * On the LHS menu, select **OAuth Apps** in **Developer settings**.
 * Click **New OAuth App**. Add the following information:
   * **Application name**: this can be anything
-  * **Homepage URL**: set this to `https://data.scrc.uk/`, for example
-  * **Authorization callback URL**: set this to `https://data.scrc.uk/accounts/github/login/callback/`, for example
+  * **Homepage URL**: set this to `https://data.fairdatapipeline.org/`, for example
+  * **Authorization callback URL**: set this to `https://data.fairdatapipeline.org/accounts/github/login/callback/`, for example
 * Click **Register Application**
 * The provided **Client ID** and **Client Secret** will be required later
 
 
 
 ## Django setup
-* Login to the admin page, e.g. https://data.scrc.uk/admin, as superuser
-* Select **Sites** and edit the **Display name** and **Domain name** so they are set correctly to the external URL (e.g. data.scrc.uk). Save the changes
+* Login to the admin page, e.g. https://data.fairdatapipeline.org/admin, as superuser
+* Select **Sites** and edit the **Display name** and **Domain name** so they are set correctly to the external URL (e.g. data.fairdatapipeline.org). Save the changes
 * Select **Social applications**.
 * Select **Add social application**. Add the following information:
   * **Provider** should be set to `GitHub`
@@ -44,5 +44,5 @@ Authorization: token <token>
 ```
 Example request:
 ```
-curl -i -X POST -H "Content-Type: application/json" -H "Authorization: token <token>" https://data.scrc.uk/api/model/
+curl -i -X POST -H "Content-Type: application/json" -H "Authorization: token <token>" https://data.fairdatapipeline.org/api/model/
 ```

docs/data_upload.md

@@ -3,7 +3,7 @@
 ## Uploading data
 There are two steps required to upload data. Firstly you need to request an upload URL. For example:
 ```
-curl -i -X POST -H "Authorization: token <token>" https://data.scrc.uk/api/data/<checksum>
+curl -i -X POST -H "Authorization: token <token>" https://data.fairdatapipeline.org/api/data/<checksum>
 ```
 where `<token>` should be replaced with a valid access token and `<checksum>` should be replaced by the SHA-1 checksum of the file you want to upload. The Linux command `sha1sum` can be used to calculate the SHA-1 checksum.
 
@@ -22,38 +22,38 @@ curl -i --upload-file <filename> "<url>"
 ```
 where `<filename>` should be replaced with the name of the file you want to upload and `<url>` should be replaced with the URL obtained in the previous step. The status code will be 201 if the file was uploaded successfully.
 
-The `StorageLocation` should be created in the usual way, using https://data.scrc.uk/api/storage_root/1/ as the `StorageRoot`, e.g. POST the following JSON to https://data.scrc.uk/api/storage_location/:
+The `StorageLocation` should be created in the usual way, using https://data.fairdatapipeline.org/api/storage_root/1/ as the `StorageRoot`, e.g. POST the following JSON to https://data.fairdatapipeline.org/api/storage_location/:
 ```
 {
   "path": "<checksum>",
   "hash": "<checksum>",
-  "storage_root": "https://data.scrc.uk/api/storage_root/1/"
+  "storage_root": "https://data.fairdatapipeline.org/api/storage_root/1/"
 }
 ```
 where for the checksum should be used for both the `path` and `hash`.
 
-Next the `Object` can be created. When creating an `Object` you should specify a `FileType`, e.g. POST the following JSON to https://data.scrc.uk/api/object/:
+Next the `Object` can be created. When creating an `Object` you should specify a `FileType`, e.g. POST the following JSON to https://data.fairdatapipeline.org/api/object/:
 ```
 {
   "description": "...",
-  "storage_location": "https://data.scrc.uk/api/storage_location/<ID>/",
-  "file_type": "https://data.scrc.uk/api/file_type/<ID>/"
+  "storage_location": "https://data.fairdatapipeline.org/api/storage_location/<ID>/",
+  "file_type": "https://data.fairdatapipeline.org/api/file_type/<ID>/"
 }
 ```
-Existing file types are listed here: https://data.scrc.uk/api/file_type/. New file types can be added by anyone if needed.
+Existing file types are listed here: https://data.fairdatapipeline.org/api/file_type/. New file types can be added by anyone if needed.
 
 ## Downloading data
 Firstly note that data can only be downloaded from the object store once the `StorageLocation` and `Object` have been created.
 
-Files can be downloaded from `https://data.scrc.uk/data/<checksum>`. This will redirect directly to the object storage.
+Files can be downloaded from `https://data.fairdatapipeline.org/data/<checksum>`. This will redirect directly to the object storage.
 
 Aliases can also be used to download data. For data products use:
 ```
-https://data.scrc.uk/data_product/<namespace>:<data product name>@<version>
+https://data.fairdatapipeline.org/data_product/<namespace>:<data product name>@<version>
 ```
 and for external objects use:
 ```
-https://data.scrc.uk/external_object/<alternate_identifier>:<title>@<version>
+https://data.fairdatapipeline.org/external_object/<alternate_identifier>:<title>@<version>
 ```
 For external objects query parameters can be used to return different URLs:
 * **source**: returns the `Source` associated with the external object

docs/local_registry.md

@@ -3,7 +3,7 @@
 ## Installation
 To initialise a local registry, run the following command from your terminal:
 ```
-/bin/bash -c "$(curl -fsSL https://data.scrc.uk/static/localregistry.sh)"
+/bin/bash -c "$(curl -fsSL https://data.fairdatapipeline.org/static/localregistry.sh)"
 ```
 This will install the registry and all the related files will be stored in `~/.scrc`.
 

drams/base_settings.py

@@ -12,8 +12,8 @@
 ALLOWED_HOSTS = ['127.0.0.1', 'localhost']
 
 # The URL of the central public registry
-CENTRAL_REGISTRY_URL = 'https://data.scrc.uk/'
-DOMAIN_URL = "https://data.scrc.uk/"
+CENTRAL_REGISTRY_URL = 'https://data.fairdatapipeline.org/'
+DOMAIN_URL = "https://data.fairdatapipeline.org/"
 
 
 # Application definition
@@ -164,3 +164,5 @@
 
 CONFIG_LOCATION = ""
 CACHE_DURATION = 0
+
+AUTHORISED_USER_FILE = ""

drams/dev-settings.py

@@ -3,4 +3,4 @@
 DEBUG = True
 
 # The URL of the central public registry
-CENTRAL_REGISTRY_URL = 'https://data.scrc.uk/'
+CENTRAL_REGISTRY_URL = 'https://data.fairdatapipeline.org/'

drams/settings.py

@@ -3,7 +3,7 @@
 with open('/home/ubuntu/secret_key.txt') as f:
     SECRET_KEY = f.read().strip()
 
-ALLOWED_HOSTS = ['data.scrc.uk', '127.0.0.1', 'localhost']
+ALLOWED_HOSTS = ['data.fairdatapipeline.org', '127.0.0.1', 'localhost']
 
 DATABASES = {
     'default': {