Feat/dos (#131)

* feat(dos): dos get endpoint

* feat(prefix): prepend prefix option

README.md

@@ -91,3 +91,11 @@ swagger-codegen generate -i openapis/swagger.yaml -l python -o swagger_client
 cd swagger_client; python setup.py develop; cd -
 py.test -v tests/
 ```
+
+## Testing with Docker
+
+Doesn't work with all the DB tests yet, but you can adjust to run specific tests as necessary.
+
+```
+docker build -t indexd -f TestDockerfile .
+```

TestDockerfile

@@ -0,0 +1,38 @@
+# A dockerfile based on our .travis.yml
+# Useful for running the tests locally 
+# in a self contained environment
+FROM python:2-slim
+
+RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main" | tee /etc/apt/sources.list.d/webupd8team-java.list \
+  && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list \
+  && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 \
+  && apt-get update && \
+  mkdir -p /usr/share/man/man1/ \
+  && apt-get install -y software-properties-common && \
+  (echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections) && \
+  apt-get install -y oracle-java8-set-default && \
+  apt-get clean && \
+  rm -fr /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install Deps
+RUN dpkg --add-architecture i386 && apt-get update && apt-get install -y --force-yes expect git wget libc6-i386 lib32stdc++6 lib32gcc1 lib32ncurses5 lib32z1 python curl libqt5widgets5
+
+# update some packages
+RUN apt-get install -y ca-certificates && update-ca-certificates
+
+RUN apt-get install -y --no-install-recommends build-essential python-dev libpq-dev libevent-dev libmagic-dev zip unzip \
+  && apt-get clean && rm -fr /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN wget https://oss.sonatype.org/content/repositories/releases/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar
+
+COPY test-requirements.txt /indexd/test-requirements.txt
+COPY requirements.txt /indexd/requirements.txt
+WORKDIR /indexd
+RUN pip install -r test-requirements.txt && pip install -r requirements.txt
+
+COPY . /indexd
+
+RUN java -jar /swagger-codegen-cli-2.3.1.jar generate -i openapis/swagger.yaml -l python -o swagger_client \
+  && cd swagger_client; python setup.py install; cd .. \
+  && python setup.py install --force \
+  && pwd && py.test -vv tests

indexd/app.py

@@ -1,6 +1,7 @@
 import flask
 from .index.blueprint import blueprint as indexd_index_blueprint
 from .alias.blueprint import blueprint as indexd_alias_blueprint
+from .dos.blueprint import blueprint as indexd_dos_blueprint
 from .blueprint import blueprint as cross_blueprint
 import os
 import sys
@@ -15,6 +16,7 @@ def app_init(app, settings=None):
     app.auth = settings['auth']
     app.register_blueprint(indexd_index_blueprint)
     app.register_blueprint(indexd_alias_blueprint)
+    app.register_blueprint(indexd_dos_blueprint)
     app.register_blueprint(cross_blueprint)
 
 

indexd/blueprint.py

@@ -65,7 +65,7 @@ def get_record(record):
             except AliasNoRecordFound:
                 if not blueprint.dist or 'no_dist' in flask.request.args:
                     raise
-                return dist_get_record(record)
+                ret = dist_get_record(record)
 
 
     return flask.jsonify(ret), 200
@@ -98,7 +98,7 @@ def dist_get_record(record):
                 'host': indexd['host'],
                 'name': indexd['name'],
             }
-            return flask.jsonify(json), 200
+            return json
 
     raise IndexNoRecordFound('no record found')
 

indexd/default_settings.py

@@ -11,7 +11,8 @@
     'driver':  SQLAlchemyIndexDriver(
         'sqlite:///index.sq3', auto_migrate=AUTO_MIGRATE,
         index_config={
-            'DEFAULT_PREFIX': 'testprefix:', 'ADD_PREFIX_ALIAS': True}
+            'DEFAULT_PREFIX': 'testprefix:', 'ADD_PREFIX_ALIAS': True,
+            'PREPEND_PREFIX': True}
     ),
 }
 

indexd/dos/blueprint.py

@@ -0,0 +1,146 @@
+import flask
+
+from indexd.blueprint import dist_get_record
+
+from indexd.errors import AuthError
+from indexd.errors import UserError
+from indexd.alias.errors import NoRecordFound as AliasNoRecordFound
+from indexd.index.errors import NoRecordFound as IndexNoRecordFound
+
+blueprint = flask.Blueprint('dos', __name__)
+
+blueprint.config = dict()
+blueprint.index_driver = None
+blueprint.alias_driver = None
+blueprint.dist = []
+
+@blueprint.route('/ga4gh/dos/v1/dataobjects/<path:record>', methods=['GET'])
+def get_dos_record(record):
+    '''
+    Returns a record from the local ids, alias, or global resolvers.
+    Returns DOS Schema
+    '''
+
+    try:
+        ret = blueprint.index_driver.get(record)
+        ret['alias'] = blueprint.index_driver.get_aliases_for_did(record)
+    except IndexNoRecordFound:
+        try:
+            ret = blueprint.index_driver.get_by_alias(record)
+            ret['alias'] = blueprint.index_driver.get_aliases_for_did(ret['did'])
+        except IndexNoRecordFound:
+            try:
+                ret = blueprint.alias_driver.get(record)
+            except AliasNoRecordFound:
+                if not blueprint.dist:
+                    raise
+                ret = dist_get_record(record)
+
+    return flask.jsonify(indexd_to_dos(ret)), 200
+
+@blueprint.route('/ga4gh/dos/v1/dataobjects/list', methods=['POST'])
+def list_dos_records():
+    '''
+    Returns a record from the local ids, alias, or global resolvers.
+    Returns DOS Schema
+    '''
+    start = flask.request.json.get('page_token')
+    limit = flask.request.json.get('page_size')
+
+    try:
+        limit = 100 if limit is None else int(limit)
+    except ValueError:
+        raise UserError('limit must be an integer')
+
+    if limit <= 0 or limit > 1024:
+        raise UserError('limit must be between 1 and 1024')
+
+    url = flask.request.json.get('url')
+
+    # Support this in the future when we have
+    # more fully featured aliases?
+    #alias = flask.request.json.get('alias')
+
+    checksum = flask.request.json.get('checksum')
+    if checksum:
+        hashes = {checksum['type']: checksum['checksum']}
+    else:
+        hashes = None
+
+    records = blueprint.index_driver.ids(
+        start=start,
+        limit=limit,
+        urls=url,
+        hashes=hashes
+    )
+
+    for record in records:
+        record['alias'] = blueprint.index_driver.get_aliases_for_did(record['did'])
+
+    ret = {"data_objects": [indexd_to_dos(record)['data_object'] for record in records]}
+
+    return flask.jsonify(ret), 200
+
+def indexd_to_dos(record):
+    data_object = {
+        "id": record['did'],
+        "name": record['file_name'],
+        'created': record['created_date'],
+        'updated': record['updated_date'],
+        "size": record['size'],
+        "version": record['rev'],
+        "description": "",
+        "mime_type": ""
+    }
+
+    data_object['aliases'] = record['alias']
+
+    # parse out checksums
+    data_object['checksums'] = []
+    for k in record['hashes']:
+        data_object['checksums'].append(
+            {'checksum': record['hashes'][k], 'type': k})
+
+    # parse out the urls
+    data_object['urls'] = []
+    for url in record['urls']:
+        url_object = {
+            'url': url }
+        if 'metadata' in record and record['metadata']:
+           url_object['system_metadata'] = record['metadata']
+        if 'urls_metadata' in record and url in record['urls_metadata'] and record['urls_metadata'][url]:
+            url_object['user_metadata'] = record['urls_metadata'][url]
+        data_object['urls'].append(url_object)
+
+    result = { "data_object": data_object }
+    return result
+
+
+@blueprint.errorhandler(UserError)
+def handle_user_error(err):
+    ret = { msg: str(err), status_code: 0 }
+    return flask.jsonify(ret), 400
+
+@blueprint.errorhandler(AuthError)
+def handle_auth_error(err):
+    ret = { msg: str(err), status_code: 0 }
+    return flask.jsonify(ret), 403
+
+@blueprint.errorhandler(AliasNoRecordFound)
+def handle_no_alias_record_error(err):
+    ret = { msg: str(err), status_code: 0 }
+    return flask.jsonify(ret), 404
+
+@blueprint.errorhandler(IndexNoRecordFound)
+def handle_no_index_record_error(err):
+    ret = { msg: str(err), status_code: 0 }
+    return flask.jsonify(ret), 404
+
+@blueprint.record
+def get_config(setup_state):
+    index_config = setup_state.app.config['INDEX']
+    alias_config = setup_state.app.config['ALIAS']
+    blueprint.index_driver = index_config['driver']
+    blueprint.alias_driver = alias_config['driver']
+    if 'DIST' in setup_state.app.config:
+        blueprint.dist = setup_state.app.config['DIST']

indexd/index/drivers/alchemy.py

@@ -452,7 +452,13 @@ def add(self,
             record.file_name = file_name
             record.version = version
 
-            record.did = did or str(uuid.uuid4())
+            if did:
+                record.did = did
+            else:
+                new_did = str(uuid.uuid4())
+                if self.config.get('PREPEND_PREFIX'):
+                    new_did = self.config['DEFAULT_PREFIX'] + new_did
+                record.did = new_did
 
             record.rev = str(uuid.uuid4())[:8]
 
@@ -517,6 +523,17 @@ def get_by_alias(self, alias):
                 raise MultipleRecordsFound('multiple records found')
             return record.to_document_dict()
 
+    def get_aliases_for_did(self, did):
+        """
+        Gets the aliases for a did
+        """
+        with self.session as session:
+            query = (
+                session.query(IndexRecordAlias)
+                .filter(IndexRecordAlias.did == did)
+            )
+            return [i.name for i in query]
+
     def get(self, did):
         """
         Gets a record given the record id or baseid.