Merge pull request #75 from uc-cdis/fix/index_creation

Fix/index creation
uc-cdis · Jan 29, 2018 · 62fae95 · 62fae95
2 parents 36bc61a + 6287263
commit 62fae95
Show file tree

Hide file tree

Showing 7 changed files with 152 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -135,6 +135,55 @@ HTTP/1.1 200 OK
 
 [Full schema for creating an index](indexd/index/schema.py)
 
+### Create an index given did
+
+POST /index/
+Content-Type: application/json
+```
+{
+  "did": "3d313755-cbb4-4b08-899d-7bbac1f6e67d",
+  "form": "object",
+  "size": 123,
+  "file_name": "abc.txt",
+  "version": "ver_123",
+  "urls": ["s3://endpointurl/bucket/key"],
+  "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"}
+}
+```
+
+| Parameters        | Values           |
+| -----:|:-----|
+| did       | Unique digital ID |
+| form      | Can be one of 'object', 'container', 'multipart' |
+| size      |  File size in bytes (commonly computed via wc -c filename) |
+| file_name |  Optional file name |
+| version | Optional version string |
+| urls      | URLs where the datafile is stored, can be multiple locations both internally and externally |
+| hashes    |  Dictionary is a string:string datastore supporting md5, sha, sha256, sha512 hash types |
+
+Curl example:
+```
+curl http://localhost/index/ -u test:test -H "Content-type: application/json" -X POST -d '{"form": "object","size": 123,"did": "3d313755-cbb4-4b08-899d-7bbac1f6e67d", urls": ["s3://endpointurl/bucket/key"],"hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"}}'
+```
+
+***Response***
+HTTP/1.1 200 OK
+```
+{
+  "did": "3d313755-cbb4-4b08-899d-7bbac1f6e67d",
+  "baseid": "703d4g20-103f-8452-a672-878vb42ef8a5"
+  "rev": "c6fc83d0"
+}
+```
+
+| Parameters        | Values           |
+| ----:|:----|
+| did     | Unique digital ID |
+| baseid  | Internal UUID assigned by the index service. All versions of a record share the same baseid |
+| rev     | 8-digit hex revision ID assigned by the index service |
+
+[Full schema for creating an index](indexd/index/schema.py)
+
 ### Update an index
 
 PUT /index/UUID?rev=REVSTRING

diff --git a/indexd/index/blueprint.py b/indexd/index/blueprint.py
@@ -176,15 +176,18 @@ def post_index_record():
     except jsonschema.ValidationError as err:
         raise UserError(err)
 
+    did = flask.request.json.get('did')
     form = flask.request.json['form']
     size = flask.request.json['size']
     urls = flask.request.json['urls']
+
     hashes = flask.request.json['hashes']
     file_name = flask.request.json.get('file_name')
     metadata = flask.request.json.get('metadata')
     version = flask.request.json.get('version')
 
     did, rev, baseid = blueprint.index_driver.add(
+        did,
         form,
         size=size,
         file_name=file_name,

diff --git a/indexd/index/driver.py b/indexd/index/driver.py
@@ -31,7 +31,7 @@ def hashes_to_urls(self, size, hashes, start=0, limit=100):
         raise NotImplementedError('TODO')
 
     @abc.abstractmethod
-    def add(self, form, size=None, urls=None, hashes=None, file_name=None, metadata=None, version=None):
+    def add(self, form, did=None, size=None, urls=None, hashes=None, file_name=None, metadata=None, version=None):
         '''
         Creates record for given data.
         '''

diff --git a/indexd/index/drivers/alchemy.py b/indexd/index/drivers/alchemy.py
@@ -255,9 +255,10 @@ def hashes_to_urls(self, size, hashes, start=0, limit=100):
 
             return [r.url for r in query]
 
-    def add(self, form, size=None, file_name=None, metadata=None, version=None, urls=None, hashes=None):
+    def add(self, form, did=None, size=None, file_name=None, metadata=None, version=None, urls=None, hashes=None):
         '''
-        Creates a new record given size, urls, hashes, metadata, and file name.
+        Creates a new record given size, urls, hashes, metadata, file name and version
+        if did is provided, update the new record with the did otherwise create it
         '''
 
         if urls is None:
@@ -266,6 +267,7 @@ def add(self, form, size=None, file_name=None, metadata=None, version=None, urls
             hashes = {}
         if metadata is None:
             metadata = {}
+
         with self.session as session:
             record = IndexRecord()
 
@@ -276,10 +278,10 @@ def add(self, form, size=None, file_name=None, metadata=None, version=None, urls
             record.baseid = baseid
             record.file_name = file_name
             record.version = version
-
 
-            did = str(uuid.uuid4())
-            record.did, record.rev = did, str(uuid.uuid4())[:8]
+            record.did = did or str(uuid.uuid4())
+
+            record.rev = str(uuid.uuid4())[:8]
 
             record.form, record.size = form, size
 
@@ -310,9 +312,12 @@ def add(self, form, size=None, file_name=None, metadata=None, version=None, urls
 
             try:
                 session.add(base_version)
+            except:
+                raise UserError('{baseid} already exists'.format(baseid=baseid), 400)
+
+            try:
                 session.add(record)
                 session.commit()
-
             except IntegrityError:
                 raise UserError('{did} already exists'.format(did=did), 400)
 
@@ -338,7 +343,7 @@ def get(self, did):
 
             form = record.form
             size = record.size
-            
+
             file_name = record.file_name
             version = record.version
 
@@ -389,10 +394,10 @@ def update(self, did, rev, urls=None, file_name=None, version=None):
                     did=record.did,
                     url=url
                 ) for url in urls]
-            
+
             if file_name is not None:
                 record.file_name = file_name
-            
+
             if version is not None:
                 record.version = version
 

diff --git a/openapis/swagger.yaml b/openapis/swagger.yaml
@@ -371,6 +371,10 @@ definitions:
       - urls
       - form
     properties:
+      did:
+        type: string
+        pattern: >-
+          ^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$
       base_id:
         type: string
         pattern: >-

diff --git a/tests/test_client.py b/tests/test_client.py
@@ -17,6 +17,55 @@ def test_index_create(client, user):
         data=json.dumps(data),
         headers=user).status_code == 200
 
+def test_index_create_with_valid_did(client, user):
+    data = {
+        'did':'3d313755-cbb4-4b08-899d-7bbac1f6e67d',
+        'form': 'object',
+        'size': 123,
+        'urls': ['s3://endpointurl/bucket/key'],
+        'hashes': {'md5': '8b9942cf415384b27cadf1f4d2d682e5'}}
+
+    assert client.post(
+        '/index/',
+        data=json.dumps(data),
+        headers=user).status_code == 200
+
+def test_index_create_with_invalid_did(client, user):
+    data = {
+        'did':'3d313755-cbb4-4b0fdfdfd8-899d-7bbac1f6e67dfdd',
+        'form': 'object',
+        'size': 123,
+        'urls': ['s3://endpointurl/bucket/key'],
+        'hashes': {'md5': '8b9942cf415384b27cadf1f4d2d682e5'}}
+
+    assert client.post(
+        '/index/',
+        data=json.dumps(data),
+        headers=user).status_code == 400
+
+def test_index_create_with_duplicate_did(client, user):
+    data = {
+        'did':'3d313755-cbb4-4b0fdfdfd8-899d-7bbac1f6e67dfdd',
+        'form': 'object',
+        'size': 123,
+        'urls': ['s3://endpointurl/bucket/key'],
+        'hashes': {'md5': '8b9942cf415384b27cadf1f4d2d682e5'}}
+    client.post(
+        '/index/',
+        data=json.dumps(data),
+        headers=user)
+
+    data2 = {
+        'did':'3d313755-cbb4-4b0fdfdfd8-899d-7bbac1f6e67dfdd',
+        'form': 'object',
+        'size': 213,
+        'urls': ['s3://endpointurl/bucket/key'],
+        'hashes': {'md5': '469942cf415384b27cadf1f4d2d682e5'}}
+
+    assert client.post(
+        '/index/',
+        data=json.dumps(data2),
+        headers=user).status_code == 400
 
 def test_index_create_with_file_name(client, user):
     data = {

diff --git a/tests/test_driver_alchemy_crud.py b/tests/test_driver_alchemy_crud.py
@@ -8,12 +8,14 @@
 from indexd.index.errors import NoRecordFound
 from indexd.index.errors import RevisionMismatch
 
+from indexd.errors import UserError
+
 from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver, IndexRecord
 
 from datetime import datetime
 
 
-# TODO check if pytest has utilities for meta-programming of tests
+#TODO check if pytest has utilities for meta-programming of tests
 
 @util.removes('index.sq3')
 def test_driver_init_does_not_create_records():
@@ -144,6 +146,35 @@ def test_driver_add_multipart_record():
         assert record[3] == 'multipart', 'record form is not multipart'
         assert record[4] == None, 'record size non-null'
 
+@util.removes('index.sq3')
+def test_driver_add_with_valid_did():
+    '''
+    Tests creation of a record with version string.
+    '''
+    driver = SQLAlchemyIndexDriver('sqlite:///index.sq3')
+
+    form = 'object'
+    did = '3d313755-cbb4-4b08-899d-7bbac1f6e67d'
+    driver.add(form, did = did)
+    with driver.session as s:
+        assert s.query(IndexRecord).first().did == did
+
+
+@util.removes('index.sq3')
+def test_driver_add_with_duplicate_did():
+    '''
+    Tests creation of a record with version string.
+    '''
+    driver = SQLAlchemyIndexDriver('sqlite:///index.sq3')
+
+    form = 'object'
+    did = '3d313755-cbb4-4b08-899d-7bbac1f6e67d'
+    driver.add(form, did = did)
+
+    with pytest.raises(UserError):
+        driver.add(form, did = did)
+
+
 @util.removes('index.sq3')
 def test_driver_add_multiple_records():
     '''