From 650ae2c178e408a0834114473024d5f61aada30e Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 17:39:33 +0000
Subject: [PATCH 01/20] Add superclass js models earlier in Grunt

---
 Gruntfile.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Gruntfile.js b/Gruntfile.js
index 6d90f5bd..78cd2d76 100644
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -100,6 +100,8 @@ module.exports = function (grunt) {
                 jsDir + '/utilities.js',
                 jsDir + '/MinervaModel.js',
                 jsDir + '/MinervaCollection.js',
+                jsDir + '/models/DatasetModel.js',
+                jsDir + '/models/SourceModel.js',
                 jsDir + '/models/**/*.js',
                 jsDir + '/collections/**/*.js',
                 jsDir + '/views/**/*.js'

From 2e266a63dba564a74212b278ad57fd4c47139dfe Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 17:41:51 +0000
Subject: [PATCH 02/20] Style consistency for wms source widget

---
 web_external/js/views/widgets/AddWmsSourceWidget.js    | 2 +-
 web_external/templates/widgets/addWmsSourceWidget.jade | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/web_external/js/views/widgets/AddWmsSourceWidget.js b/web_external/js/views/widgets/AddWmsSourceWidget.js
index d267e3d1..c17ea7c9 100644
--- a/web_external/js/views/widgets/AddWmsSourceWidget.js
+++ b/web_external/js/views/widgets/AddWmsSourceWidget.js
@@ -1,5 +1,5 @@
 /**
-* This widget displays a form for adding WMS services
+* This widget displays a form for adding a WMS source.
 */
 minerva.views.AddWmsSourceWidget = minerva.View.extend({
 
diff --git a/web_external/templates/widgets/addWmsSourceWidget.jade b/web_external/templates/widgets/addWmsSourceWidget.jade
index f1fd9d50..743c1656 100644
--- a/web_external/templates/widgets/addWmsSourceWidget.jade
+++ b/web_external/templates/widgets/addWmsSourceWidget.jade
@@ -24,4 +24,4 @@
         a.btn.btn-small.btn-default(data-dismiss="modal") Cancel
         button.m-add-service-button.btn.btn-small.btn-primary
             i.icon-plus-squared
-            | Add WMS Service
+            | Add WMS Source

From 57c5973a950af8ee78baf17e02c517f7c39ba1ca Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 19:36:13 +0000
Subject: [PATCH 03/20] Add create elasticsearch source endpoint

---
 development.md                      | 26 ++++++++++++
 server/loader.py                    | 10 ++++-
 server/rest/elasticsearch_source.py | 65 +++++++++++++++++++++++++++++
 3 files changed, 99 insertions(+), 2 deletions(-)
 create mode 100644 development.md
 create mode 100644 server/rest/elasticsearch_source.py

diff --git a/development.md b/development.md
new file mode 100644
index 00000000..7e6b523f
--- /dev/null
+++ b/development.md
@@ -0,0 +1,26 @@
+# Minerva developers guide
+
+## Adding a source
+
+I think it's easier to work on the backend first, as you can test the api
+independently of the client.  We'll work through an example using the
+Elasticsearch source.
+
+### Source api
+
+Create an endpoint to create the source, copy server/rest/wms_source.py to
+server/rest/elasticsearch_source.py and modify accordingly.
+
+Important points are
+
+  * use the access decorator to ensure only logged in users can call the endpoint
+  * create minerva_metadata with the correct `source_type`
+  * save the source using the superclass method createSource
+  * return the document corresponding to the new source
+  * here we store the authentication credentials after encryption
+  * set the description object to display the params correctly on the swagger api page
+
+Add the endpoint to server/loader.py
+
+    info['apiRoot'].minerva_source_elasticsearch = elasticsearch_source.ElasticsearchSource()
+
diff --git a/server/loader.py b/server/loader.py
index a2c0e387..51fb1cf5 100644
--- a/server/loader.py
+++ b/server/loader.py
@@ -28,8 +28,7 @@
 
 from girder.plugins.minerva.rest import \
         analysis, dataset, s3_dataset, session, shapefile, geocode, source, \
-        wms_dataset, wms_source, geojson_dataset
-from girder.plugins.minerva.constants import PluginSettings
+        wms_dataset, wms_source, geojson_dataset, elasticsearch_source
 from girder.plugins.minerva.utility.minerva_utility import decryptCredentials
 
 
@@ -187,8 +186,15 @@ def load(info):
     info['apiRoot'].minerva_analysis = analysis.Analysis()
     info['apiRoot'].minerva_session = session.Session()
     info['apiRoot'].minerva_dataset_s3 = s3_dataset.S3Dataset()
+
     info['apiRoot'].minerva_source = source.Source()
+
     info['apiRoot'].minerva_source_wms = wms_source.WmsSource()
     info['apiRoot'].minerva_dataset_wms = wms_dataset.WmsDataset()
+
     info['apiRoot'].minerva_dataset_geojson = geojson_dataset.GeojsonDataset()
+
+    info['apiRoot'].minerva_source_elasticsearch = \
+        elasticsearch_source.ElasticsearchSource()
+
     info['serverRoot'].wms_proxy = WmsProxy()
diff --git a/server/rest/elasticsearch_source.py b/server/rest/elasticsearch_source.py
new file mode 100644
index 00000000..914ff860
--- /dev/null
+++ b/server/rest/elasticsearch_source.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
+from girder.api import access
+from girder.api.describe import Description
+from girder.api.rest import getUrlParts
+
+from girder.plugins.minerva.rest.source import Source
+from girder.plugins.minerva.utility.minerva_utility import encryptCredentials
+
+
+class ElasticsearchSource(Source):
+
+    def __init__(self):
+        self.resourceName = 'minerva_source_elasticsearch'
+        self.route('POST', (), self.createElasticsearchSource)
+
+    @access.user
+    def createElasticsearchSource(self, params):
+        name = params['name']
+        baseURL = params['baseURL']
+        parsedUrl = getUrlParts(baseURL)
+        hostName = parsedUrl.netloc
+        table = params['table']
+        username = params['username'] if 'username' in params else None
+        password = params['password'] if 'password' in params else None
+        minerva_metadata = {
+            'source_type': 'elasticsearch',
+            'table': table,
+            'elasticsearch_params': {
+                'base_url': baseURL,
+                'host_name': hostName
+            }
+        }
+        if username and password:
+            enc_creds = encryptCredentials("{}:{}".format(
+                username, password))
+            minerva_metadata['elasticsearch_params']['credentials'] = enc_creds
+        desc = 'elasticsearch source for  %s' % name
+        return self.createSource(name, minerva_metadata, desc)
+    createElasticsearchSource.description = (
+        Description('Create a source from an external elasticsearch server.')
+        .responseClass('Item')
+        .param('name', 'The name of the elasticsearch source', required=True)
+        .param('baseURL', 'URL of the elasticsearch service', required=True)
+        .param('table', 'Table of interest', required=True)
+        .param('username', 'Elasticsearch username', required=False)
+        .param('password', 'Elasticsearch password', required=False)
+        .errorResponse('Write permission denied on the source folder.', 403))

From 1cef045a7786d4fd2b5219ae6f650ab72f9a0bc0 Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 19:51:45 +0000
Subject: [PATCH 04/20] Add elasticsearch_source endpoint test

---
 development.md                      | 38 +++++++++++++
 plugin.cmake                        |  1 +
 plugin_tests/elasticsearch_test.py  | 88 +++++++++++++++++++++++++++++
 server/rest/elasticsearch_source.py |  2 +-
 4 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100644 plugin_tests/elasticsearch_test.py

diff --git a/development.md b/development.md
index 7e6b523f..0d0c552a 100644
--- a/development.md
+++ b/development.md
@@ -24,3 +24,41 @@ Add the endpoint to server/loader.py
 
     info['apiRoot'].minerva_source_elasticsearch = elasticsearch_source.ElasticsearchSource()
 
+### Testing the source api
+
+Create a test, copy plugin_tests/wms_test.py to plugin_tests/elasticsearch_test.py.
+
+Add the test to plugin.cmake
+
+    add_python_test(elasticsearch PLUGIN minerva)
+
+Run `cmake PATH_TO_GIRDER_DIR` again in your build directory to pick up the new test.
+
+Now you should see the new test in your build directory
+
+    ctest -N | grep minerva
+      Test #110: server_minerva.dataset
+      Test #111: server_minerva.source
+      Test #112: server_minerva.session
+      Test #113: server_minerva.analysis
+      Test #114: server_minerva.geonames
+      Test #115: server_minerva.s3_dataset
+      Test #116: server_minerva.import_analyses
+      Test #117: server_minerva.contour_analysis
+      Test #118: server_minerva.wms
+      Test #119: server_minerva.elasticsearch
+      Test #120: server_minerva.geojson
+      Test #121: server_minerva.mean_contour_analysis
+      Test #122: pep8_style_minerva_constants
+      Test #123: pep8_style_minerva_geonames
+      Test #124: pep8_style_minerva_rest
+      Test #125: pep8_style_minerva_utility
+      Test #126: pep8_style_minerva_bsve
+      Test #127: pep8_style_minerva_jobs
+      Test #128: jshint_minerva
+      Test #129: jsstyle_minerva
+      Test #130: web_client_minerva
+
+You can run the test, with extra verbosity
+
+    ctest -R server_minerva.elasticsearch -VV
diff --git a/plugin.cmake b/plugin.cmake
index e79493f8..d576aae5 100644
--- a/plugin.cmake
+++ b/plugin.cmake
@@ -23,6 +23,7 @@ add_python_test(s3_dataset PLUGIN minerva)
 add_python_test(import_analyses PLUGIN minerva)
 add_python_test(contour_analysis PLUGIN minerva)
 add_python_test(wms PLUGIN minerva)
+add_python_test(elasticsearch PLUGIN minerva)
 add_python_test(geojson PLUGIN minerva)
 
 
diff --git a/plugin_tests/elasticsearch_test.py b/plugin_tests/elasticsearch_test.py
new file mode 100644
index 00000000..d9b884a6
--- /dev/null
+++ b/plugin_tests/elasticsearch_test.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
+import os
+
+# Need to set the environment variable before importing girder
+os.environ['GIRDER_PORT'] = os.environ.get('GIRDER_TEST_PORT', '20200')  # noqa
+
+from tests import base
+
+
+def setUpModule():
+    """
+    Enable the minerva plugin and start the server.
+    """
+    base.enabledPlugins.append('jobs')
+    base.enabledPlugins.append('romanesco')
+    base.enabledPlugins.append('gravatar')
+    base.enabledPlugins.append('minerva')
+    base.startServer(False)
+
+
+def tearDownModule():
+    """
+    Stop the server.
+    """
+    base.stopServer()
+
+
+class ElasticsearchTestCase(base.TestCase):
+
+    """
+    Tests of the minerva source API endpoints.
+    """
+
+    def setUp(self):
+        """
+        Set up the test case with  a user
+        """
+        super(ElasticsearchTestCase, self).setUp()
+
+        self._user = self.model('user').createUser(
+            'minervauser', 'password', 'minerva', 'user',
+            'minervauser@example.com')
+
+    def testCreateElasticsearchSource(self):
+        """
+        Test the minerva ELASTICSEARCH source API endpoints.
+        """
+
+        path = '/minerva_source_elasticsearch'
+        name = 'testElasticsearch'
+        username = ''
+        password = ''
+        baseURL = 'http://elasticsearch.com'
+        table = 'mytable'
+        params = {
+            'name': name,
+            'username': username,
+            'password': password,
+            'table': table,
+            'baseURL': baseURL
+        }
+        response = self.request(path=path, method='POST', params=params, user=self._user)
+        self.assertStatusOk(response)
+        elasticsearchSource = response.json
+        print(response.json)
+        minerva_metadata = elasticsearchSource['meta']['minerva']
+        self.assertEquals(elasticsearchSource['name'], name, 'incorrect elasticsearch source name')
+        self.assertEquals(minerva_metadata['source_type'], 'elasticsearch', 'incorrect elasticsearch source type')
+        self.assertEquals(minerva_metadata['elasticsearch_params']['base_url'], baseURL, 'incorrect elasticsearch source baseURL')
+        self.assertEquals(minerva_metadata['elasticsearch_params']['table'], table, 'incorrect elasticsearch source table')
diff --git a/server/rest/elasticsearch_source.py b/server/rest/elasticsearch_source.py
index 914ff860..d2b62b6d 100644
--- a/server/rest/elasticsearch_source.py
+++ b/server/rest/elasticsearch_source.py
@@ -42,8 +42,8 @@ def createElasticsearchSource(self, params):
         password = params['password'] if 'password' in params else None
         minerva_metadata = {
             'source_type': 'elasticsearch',
-            'table': table,
             'elasticsearch_params': {
+                'table': table,
                 'base_url': baseURL,
                 'host_name': hostName
             }

From 9c1aacfadbabb3bf6abc4dfb33448ddecda6a6d0 Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 19:58:23 +0000
Subject: [PATCH 05/20] Add elasticsearch source type to SourceCollection

---
 development.md                                | 10 +++++++++
 .../js/collections/SourceCollection.js        | 21 +++++++++++--------
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/development.md b/development.md
index 0d0c552a..2cfb0820 100644
--- a/development.md
+++ b/development.md
@@ -62,3 +62,13 @@ Now you should see the new test in your build directory
 You can run the test, with extra verbosity
 
     ctest -R server_minerva.elasticsearch -VV
+
+
+### Add the source to client side collection
+
+Add the new source type to web/external/js/collections/SourceCollection.js,
+this will prevent mysterious backbone errors later on like
+
+    `a.on is not a function`
+
+You're welcome.
diff --git a/web_external/js/collections/SourceCollection.js b/web_external/js/collections/SourceCollection.js
index 5759f0fb..6843fa82 100644
--- a/web_external/js/collections/SourceCollection.js
+++ b/web_external/js/collections/SourceCollection.js
@@ -5,16 +5,19 @@ minerva.collections.SourceCollection = minerva.collections.MinervaCollection.ext
             if (attrs.meta.minerva.source_type === 'wms') {
                 return new minerva.models.WmsSourceModel(attrs, options);
             }
-        } else {
-            console.error('Source collection includes unknown source type');
-            console.error(attrs);
-            girder.events.trigger('g:alert', {
-                icon: 'cancel',
-                text: 'Unknown source type in collection.',
-                type: 'error',
-                timeout: 4000
-            });
+            else if (attrs.meta.minerva.source_type === 'elasticsearch') {
+                return new minerva.models.ElasticsearchSourceModel(attrs, options);
+            }
         }
+
+        console.error('Source collection includes unknown source type');
+        console.error(attrs);
+        girder.events.trigger('g:alert', {
+            icon: 'cancel',
+            text: 'Unknown source type in collection.',
+            type: 'error',
+            timeout: 4000
+        });
     },
 
     path: 'minerva_source',

From 06644db1f9bed4182316a79678584928a073cc87 Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 20:00:04 +0000
Subject: [PATCH 06/20] Add note about testing api endpoint via swagger page

---
 development.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/development.md b/development.md
index 2cfb0820..5f644fe3 100644
--- a/development.md
+++ b/development.md
@@ -24,6 +24,11 @@ Add the endpoint to server/loader.py
 
     info['apiRoot'].minerva_source_elasticsearch = elasticsearch_source.ElasticsearchSource()
 
+You should now be able to see your endpoint through the swagger api page, and
+test it there.  Usually
+
+    http://localhost:8080/api
+
 ### Testing the source api
 
 Create a test, copy plugin_tests/wms_test.py to plugin_tests/elasticsearch_test.py.

From d26a3c5b3c76c38f14fa0b306f7005a2dafbe16d Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 20:12:35 +0000
Subject: [PATCH 07/20] Add elasticsearch source model

---
 development.md                                |  4 ++++
 .../js/models/ElasticsearchSourceModel.js     | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 web_external/js/models/ElasticsearchSourceModel.js

diff --git a/development.md b/development.md
index 5f644fe3..1cd3b69b 100644
--- a/development.md
+++ b/development.md
@@ -77,3 +77,7 @@ this will prevent mysterious backbone errors later on like
     `a.on is not a function`
 
 You're welcome.
+
+### Add a new source model
+
+Add a new model like web_external/js/models/ElasticsearchSourceModel.js.
diff --git a/web_external/js/models/ElasticsearchSourceModel.js b/web_external/js/models/ElasticsearchSourceModel.js
new file mode 100644
index 00000000..15a44415
--- /dev/null
+++ b/web_external/js/models/ElasticsearchSourceModel.js
@@ -0,0 +1,19 @@
+minerva.models.ElasticsearchSourceModel = minerva.models.SourceModel.extend({
+
+    createSource: function (params) {
+        girder.restRequest({
+            path: '/minerva_source_elasticsearch',
+            type: 'POST',
+            data: params,
+            error: null // ignore default error behavior (validation may fail)
+        }).done(_.bind(function (resp) {
+            this.set(resp);
+            this.trigger('m:sourceReceived');
+        }, this)).error(_.bind(function (err) {
+            this.trigger('m:error', err);
+        }, this));
+
+        return this;
+    }
+
+});

From 9916d34102828451821cb4c41e2c3667cb7dd8fd Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 20:16:21 +0000
Subject: [PATCH 08/20] Add addElasticsearchSourceWidget

---
 development.md                                | 15 ++++++++
 .../widgets/AddElasticsearchSourceWidget.js   | 35 +++++++++++++++++++
 .../js/views/widgets/AddSourceWidget.js       |  8 +++++
 .../widgets/addElasticsearchSourceWidget.jade | 30 ++++++++++++++++
 .../templates/widgets/addSourceWidget.jade    |  5 +++
 5 files changed, 93 insertions(+)
 create mode 100644 web_external/js/views/widgets/AddElasticsearchSourceWidget.js
 create mode 100644 web_external/templates/widgets/addElasticsearchSourceWidget.jade

diff --git a/development.md b/development.md
index 1cd3b69b..fb5acfb2 100644
--- a/development.md
+++ b/development.md
@@ -81,3 +81,18 @@ You're welcome.
 ### Add a new source model
 
 Add a new model like web_external/js/models/ElasticsearchSourceModel.js.
+
+### Add the source to AddSourceWidget
+
+Add your new source type as an option in web_external/templates/widgets/addSourceWidget.jade
+and deal with the new option in the `submit #m-add-source-form` event handler in
+web_external/js/views/widgets/AddSourceWidget.js by creating a new add widget specific
+to your source, e.g. `AddElasticsearchSourceWidget`.
+
+Test that when you click on the add new source icon in the source panel, your
+new source type is displayed as an option.
+
+Create the widget to add your new source type, e.g. in
+
+    web_external/js/views/widgets/AddElasticsearchSourceWidget.js
+    web_external/templates/widgets/addElasticsearchSourceWidget.jade
diff --git a/web_external/js/views/widgets/AddElasticsearchSourceWidget.js b/web_external/js/views/widgets/AddElasticsearchSourceWidget.js
new file mode 100644
index 00000000..10ed6d7e
--- /dev/null
+++ b/web_external/js/views/widgets/AddElasticsearchSourceWidget.js
@@ -0,0 +1,35 @@
+/**
+* This widget displays a form for adding an Elasticsearch source.
+*/
+minerva.views.AddElasticsearchSourceWidget = minerva.View.extend({
+
+    events: {
+        'submit #m-add-elasticsearch-source-form': function (e) {
+            e.preventDefault();
+            var params = {
+                name:     this.$('#m-elasticsearch-name').val(),
+                baseURL:  this.$('#m-elasticsearch-uri').val(),
+                table:  this.$('#m-elasticsearch-table').val(),
+                username: this.$('#m-elasticsearch-username').val(),
+                password: this.$('#m-elasticsearch-password').val()
+            };
+            var elasticsearchSource = new minerva.models.ElasticsearchSourceModel({});
+            elasticsearchSource.on('m:sourceReceived', function () {
+                this.$el.modal('hide');
+                // TODO: might need to be added to a new panel/data sources ?
+                this.collection.add(elasticsearchSource);
+            }, this).createSource(params);
+        }
+    },
+
+    initialize: function (settings) {
+        this.collection = settings.collection;
+    },
+
+    render: function () {
+        var modal = this.$el.html(minerva.templates.addElasticsearchSourceWidget({}));
+        modal.trigger($.Event('ready.girder.modal', {relatedTarget: modal}));
+        return this;
+    }
+
+});
diff --git a/web_external/js/views/widgets/AddSourceWidget.js b/web_external/js/views/widgets/AddSourceWidget.js
index d4335206..cd02c2f0 100644
--- a/web_external/js/views/widgets/AddSourceWidget.js
+++ b/web_external/js/views/widgets/AddSourceWidget.js
@@ -17,6 +17,14 @@ minerva.views.AddSourceWidget = minerva.View.extend({
                     collection: this.collection,
                     parentView: this.parentView
                 }).render();
+            } else if (sourceType === 'm-elasticsearch-source') {
+                this.elasticsearchSourceWidget = new minerva.views.AddElasticsearchSourceWidget({
+                    el: container,
+                    title: 'Enter Elasticsearch Source details',
+                    noParent: true,
+                    collection: this.collection,
+                    parentView: this.parentView
+                }).render();
             } else {
                 console.error('Unknown source type');
             }
diff --git a/web_external/templates/widgets/addElasticsearchSourceWidget.jade b/web_external/templates/widgets/addElasticsearchSourceWidget.jade
new file mode 100644
index 00000000..48ed6be6
--- /dev/null
+++ b/web_external/templates/widgets/addElasticsearchSourceWidget.jade
@@ -0,0 +1,30 @@
+.modal-dialog
+  .modal-content
+    form#m-add-elasticsearch-source-form.modal-form(role="form")
+      .modal-header
+        button.close(data-dismiss="modal", aria-hidden="true", type="button") &times;
+        h4.modal-title
+          | Enter Elasticsearch Service details
+
+      .modal-body
+        .form-group
+          label.control-label(for="m-elasticsearch-name") Name
+          input.input-sm#m-elasticsearch-name.form-control(type="text", placeholder="Enter Elasticsearch name")
+        .form-group
+          label.control-label(for="m-elasticsearch-uri") Elasticsearch baseURL
+          input.input-sm#m-elasticsearch-uri.form-control(type="text", placeholder="http://elasticsearch.com")
+        .form-group
+          label.control-label(for="m-elasticsearch-table") Elasticsearch table
+          input.input-sm#m-elasticsearch-table.form-control(type="text", placeholder="table")
+        .form-group
+          label.control-label(for="m-elasticsearch-username") Username
+          input.input-sm#m-elasticsearch-username.form-control(type="text", placeholder="username")
+        .form-group
+          label.control-label(for="m-elasticsearch-password") Password
+          input.input-sm#m-elasticsearch-password.form-control(type="password", placeholder="password")
+
+      .modal-footer
+        a.btn.btn-small.btn-default(data-dismiss="modal") Cancel
+        button.m-add-service-button.btn.btn-small.btn-primary
+            i.icon-plus-squared
+            | Add Elasticsearch Source
diff --git a/web_external/templates/widgets/addSourceWidget.jade b/web_external/templates/widgets/addSourceWidget.jade
index b36ed19e..80fee72c 100644
--- a/web_external/templates/widgets/addSourceWidget.jade
+++ b/web_external/templates/widgets/addSourceWidget.jade
@@ -12,6 +12,11 @@
             label
               i.icon-layers.m-add-source-type-icon
               | WMS Source
+          .radio
+            input#m-elasticsearch-source(type="radio", name="m-source-add-method")
+            label
+              i.icon-search.m-add-source-type-icon
+              | Elasticsearch Source
       .modal-footer
         a.btn.btn-small.btn-default(data-dismiss="modal") Cancel
         button.m-add-source-button.btn.btn-small.btn-primary

From 2e12d6e19ef7944f22e7a739c87d8e9ff7aaf796 Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 20:26:46 +0000
Subject: [PATCH 09/20] Add elasticsearch source to source panel display

---
 development.md                                 |  7 +++++++
 web_external/stylesheets/body/sourcePanel.styl |  3 +++
 web_external/templates/body/sourcePanel.jade   | 16 ++++++++++------
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/development.md b/development.md
index fb5acfb2..b175ad9c 100644
--- a/development.md
+++ b/development.md
@@ -96,3 +96,10 @@ Create the widget to add your new source type, e.g. in
 
     web_external/js/views/widgets/AddElasticsearchSourceWidget.js
     web_external/templates/widgets/addElasticsearchSourceWidget.jade
+
+### Display the new source in the source panel
+
+Update the necessary in
+
+    web_external/templates/body/sourcePanel.jade
+    web_external/stylesheets/body/sourcePanel.styl
diff --git a/web_external/stylesheets/body/sourcePanel.styl b/web_external/stylesheets/body/sourcePanel.styl
index f20c3ee0..891df0aa 100644
--- a/web_external/stylesheets/body/sourcePanel.styl
+++ b/web_external/stylesheets/body/sourcePanel.styl
@@ -14,6 +14,9 @@
     .m-display-wms-layers-list
         float: left
 
+    .m-display-elasticsearch-query
+        float: left
+
     .m-delete-source
         float right
 
diff --git a/web_external/templates/body/sourcePanel.jade b/web_external/templates/body/sourcePanel.jade
index ca8c32da..b37c7799 100644
--- a/web_external/templates/body/sourcePanel.jade
+++ b/web_external/templates/body/sourcePanel.jade
@@ -11,10 +11,14 @@
                         - var attributes = {'cid': source.cid}
                         if source.getSourceType() === 'wms'
                             //- list icon to open the list of WMS layers
-                            - var classes = 'icon-list m-icon-enabled m-display-wms-layers-list'
+                            - var classes = 'icon-layers m-icon-enabled m-display-wms-layers-list'
                             i(class=classes)&attributes(attributes)
-                            //- trash icon to delete source
-                            - var classes = 'icon-trash m-icon-enabled m-delete-source'
-                            i(title='delete source', class=classes)&attributes(attributes)
-                            //- info icon for minerva metadata display
-                            i(title='display source info').icon-info-circled.m-icon-info.m-icon-enabled&attributes(attributes)
+                        else if source.getSourceType() === 'elasticsearch'
+                            //- list icon to run an elasticsearch query
+                            - var classes = 'icon-search m-icon-enabled m-display-elasticsearch-query'
+                            i(class=classes)&attributes(attributes)
+                        //- trash icon to delete source
+                        - var classes = 'icon-trash m-icon-enabled m-delete-source'
+                        i(title='delete source', class=classes)&attributes(attributes)
+                        //- info icon for minerva metadata display
+                        i(title='display source info').icon-info-circled.m-icon-info.m-icon-enabled&attributes(attributes)

From 3c36ed077a01bbb78102410967d130426a169322 Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 20:39:24 +0000
Subject: [PATCH 10/20] Style compliance

---
 development.md                                         | 10 ++++++++++
 web_external/js/collections/SourceCollection.js        |  3 +--
 .../js/views/widgets/AddElasticsearchSourceWidget.js   |  1 -
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/development.md b/development.md
index b175ad9c..4d690702 100644
--- a/development.md
+++ b/development.md
@@ -68,6 +68,9 @@ You can run the test, with extra verbosity
 
     ctest -R server_minerva.elasticsearch -VV
 
+Also check your python style, and fix any errors
+
+    ctest -R pep8_style_minerva_rest -VV
 
 ### Add the source to client side collection
 
@@ -103,3 +106,10 @@ Update the necessary in
 
     web_external/templates/body/sourcePanel.jade
     web_external/stylesheets/body/sourcePanel.styl
+
+### Comply with javascript styles
+
+Because it's the law of the land.
+
+    ctest -R jshint_minerva -VV
+    ctest -R jsstyle_minerva -VV
diff --git a/web_external/js/collections/SourceCollection.js b/web_external/js/collections/SourceCollection.js
index 6843fa82..83d1d714 100644
--- a/web_external/js/collections/SourceCollection.js
+++ b/web_external/js/collections/SourceCollection.js
@@ -4,8 +4,7 @@ minerva.collections.SourceCollection = minerva.collections.MinervaCollection.ext
         if (attrs.meta && ('minerva' in attrs.meta)) {
             if (attrs.meta.minerva.source_type === 'wms') {
                 return new minerva.models.WmsSourceModel(attrs, options);
-            }
-            else if (attrs.meta.minerva.source_type === 'elasticsearch') {
+            } else if (attrs.meta.minerva.source_type === 'elasticsearch') {
                 return new minerva.models.ElasticsearchSourceModel(attrs, options);
             }
         }
diff --git a/web_external/js/views/widgets/AddElasticsearchSourceWidget.js b/web_external/js/views/widgets/AddElasticsearchSourceWidget.js
index 10ed6d7e..496c3cd4 100644
--- a/web_external/js/views/widgets/AddElasticsearchSourceWidget.js
+++ b/web_external/js/views/widgets/AddElasticsearchSourceWidget.js
@@ -16,7 +16,6 @@ minerva.views.AddElasticsearchSourceWidget = minerva.View.extend({
             var elasticsearchSource = new minerva.models.ElasticsearchSourceModel({});
             elasticsearchSource.on('m:sourceReceived', function () {
                 this.$el.modal('hide');
-                // TODO: might need to be added to a new panel/data sources ?
                 this.collection.add(elasticsearchSource);
             }, this).createSource(params);
         }

From 180928ab554390eb0680318fe95ffb7b30a40924 Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Fri, 9 Oct 2015 20:46:11 +0000
Subject: [PATCH 11/20] Add glossary to developer's guide

---
 development.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/development.md b/development.md
index 4d690702..f142db26 100644
--- a/development.md
+++ b/development.md
@@ -1,5 +1,22 @@
 # Minerva developers guide
 
+## Glossary
+
+### Source
+
+A source produces data.  A source itself cannot be visualized, but a source
+can create a dataset that can be visualized, or it can be the input to an analysis,
+which then creates a dataset that can be visualized.
+
+### Dataset
+
+A dataset contains data and can be visualized, either on a map or through some other means.
+
+### Analysis
+
+An analysis creates a dataset, but running some client side or server side process, and
+potentially using datasets and sources as inputs.
+
 ## Adding a source
 
 I think it's easier to work on the backend first, as you can test the api

From 0d3d15912859935ebb9ea1ec177546fde15e6508 Mon Sep 17 00:00:00 2001
From: Michael Grauer <michael.grauer@kitware.com>
Date: Sat, 10 Oct 2015 00:37:34 +0000
Subject: [PATCH 12/20] Add elasticsearch source action to run a query

---
 development.md                                | 18 ++++++
 web_external/js/views/body/SourcePanel.js     | 17 +++++-
 .../js/views/widgets/ElasticsearchWidget.js   | 58 +++++++++++++++++++
 .../widgets/elasticsearchWidget.jade          | 20 +++++++
 4 files changed, 112 insertions(+), 1 deletion(-)
 create mode 100644 web_external/js/views/widgets/ElasticsearchWidget.js
 create mode 100644 web_external/templates/widgets/elasticsearchWidget.jade

diff --git a/development.md b/development.md
index f142db26..d390d93e 100644
--- a/development.md
+++ b/development.md
@@ -124,6 +124,24 @@ Update the necessary in
     web_external/templates/body/sourcePanel.jade
     web_external/stylesheets/body/sourcePanel.styl
 
+### Add an action to the source displayed in the source panel
+
+If it makes sense for your source to have an action, as when there is a natural
+path to create a dataset from your source, add an action to the source displayed
+in the source panel.
+
+E.g., a WMS source naturally creates datasets by exposing a set of WMS layers
+and allowing one or more to be created as a dataset.  An Elasticsearch source naturally
+creates datasets by running an analysis which is a search query, resulting in a
+JSON dataset with a default visualization as GeoJson.
+
+Add an event handler for your source icon in web_external/js/views/body/SourcePanel.js .
+
+Add the widget constructed and rendered by the event handler
+
+    web_external/js/views/widgets/ElasticsearchWidget.js
+    web_external/templates/widgets/elasticsearchWidget.jade
+
 ### Comply with javascript styles
 
 Because it's the law of the land.
diff --git a/web_external/js/views/body/SourcePanel.js b/web_external/js/views/body/SourcePanel.js
index 95fadd7a..04953d2a 100644
--- a/web_external/js/views/body/SourcePanel.js
+++ b/web_external/js/views/body/SourcePanel.js
@@ -4,7 +4,8 @@ minerva.views.SourcePanel = minerva.View.extend({
         'click .m-add-source': 'addSourceDialog',
         'click .m-display-wms-layers-list': 'displayWmsLayersList',
         'click .m-icon-info': 'displaySourceInfo',
-        'click .m-delete-source': 'deleteSource'
+        'click .m-delete-source': 'deleteSource',
+        'click .m-display-elasticsearch-query': 'displayElasticsearchQuery'
     },
 
     addSourceDialog: function () {
@@ -51,6 +52,20 @@ minerva.views.SourcePanel = minerva.View.extend({
         source.destroy();
     },
 
+    displayElasticsearchQuery: function (evt) {
+        var el = $(evt.currentTarget);
+        var source = this.sourceCollection.get(el.attr('cid'));
+        if (!this.elasticsearchWidget) {
+            this.elasticsearchWidget = new minerva.views.ElasticsearchWidget({
+                el: $('#g-dialog-container'),
+                source: source,
+                collection: this.datasetCollection,
+                parentView: this
+            });
+        }
+        this.elasticsearchWidget.render();
+    },
+
     initialize: function (settings) {
         this.session = settings.session;
         this.sourceCollection = settings.sourceCollection;
diff --git a/web_external/js/views/widgets/ElasticsearchWidget.js b/web_external/js/views/widgets/ElasticsearchWidget.js
new file mode 100644
index 00000000..529e782f
--- /dev/null
+++ b/web_external/js/views/widgets/ElasticsearchWidget.js
@@ -0,0 +1,58 @@
+/**
+* This widget is used to display and run an Elasticsearch query.
+*/
+minerva.views.ElasticsearchWidget = minerva.View.extend({
+
+    events: {
+        'submit #m-elasticsearch-form': function (e) {
+            e.preventDefault();
+            this.$('.g-validation-failed-message').text('');
+
+            var searchParams = this.$('#m-elasticsearch-dataset-params').val();
+            var datasetName = this.$('#m-elasticsearch-dataset-name').val();
+
+            if (!datasetName || datasetName === '') {
+                this.$('.g-validation-failed-message').text('Dataset name is required');
+                return;
+            }
+
+            try {
+                // parse for side effect of validation
+                JSON.parse(searchParams);
+                this.$('.g-validation-failed-message').text('');
+            } catch (err) {
+                this.$('.g-validation-failed-message').text('Search Params must be valid JSON');
+                return;
+            }
+
+            this.$('button.m-run-elasticsearch-query').addClass('disabled');
+
+            var data = {
+                datasetName: datasetName,
+                searchParams: searchParams
+            };
+
+            girder.restRequest({
+                path: 'minerva_elasticsearch_query',
+                type: 'POST',
+                data: data
+            }).done(_.bind(function () {
+                girder.events.trigger('m:job.created');
+                this.$el.modal('hide');
+            }, this));
+
+        }
+    },
+
+    initialize: function () {
+    },
+
+    render: function () {
+        var modal = this.$el.html(minerva.templates.elasticsearchWidget({
+        })).girderModal(this).on('ready.girder.modal', _.bind(function () {
+        }, this));
+        modal.trigger($.Event('ready.girder.modal', {relatedTarget: modal}));
+
+        return this;
+    }
+});
diff --git a/web_external/templates/widgets/elasticsearchWidget.jade b/web_external/templates/widgets/elasticsearchWidget.jade
new file mode 100644
index 00000000..e5937eef
--- /dev/null
+++ b/web_external/templates/widgets/elasticsearchWidget.jade
@@ -0,0 +1,20 @@
+.modal-dialog
+  .modal-content
+    form#m-elasticsearch-form.modal-form(role="form")
+        .modal-header
+          button.close(data-dismiss="modal", aria-hidden="true", type="button") &times;
+          h4.modal-title
+            | Elasticsearch Query
+        .modal-body
+            .form-group.m-elasticsearch-dataset-name-group
+                label.control-label(for='m-elasticsearch-dataset-name') Output dataset name
+                input.input-sm#m-elasticsearch-dataset-name.form-control(type="text")
+            .form-group.m-elasticsearch-dataset-params-group
+                label.control-label(for="m-elasticsearch-dataset-params") JSON search params
+                textarea#m-elasticsearch-dataset-params.form-control(rows=20)
+            .g-validation-failed-message
+        .modal-footer
+            a.btn.btn-small.btn-default(data-dismiss="modal") Cancel
+            button.m-run-elasticsearch-query.btn.btn-small.btn-primary(type="submit")
+                i.icon-play-circled
+                | Run

From dbb410a977a1658e5078e5a2f8458518ca38b9dd Mon Sep 17 00:00:00 2001
From: Dan LaManna <me@danlamanna.com>
Date: Sat, 10 Oct 2015 18:16:39 -0400
Subject: [PATCH 13/20] Rename table/index to conform to elasticsearch
 nomenclature

---
 plugin_tests/elasticsearch_test.py                          | 6 +++---
 server/rest/elasticsearch_source.py                         | 6 +++---
 .../js/views/widgets/AddElasticsearchSourceWidget.js        | 2 +-
 .../templates/widgets/addElasticsearchSourceWidget.jade     | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/plugin_tests/elasticsearch_test.py b/plugin_tests/elasticsearch_test.py
index d9b884a6..52bef909 100644
--- a/plugin_tests/elasticsearch_test.py
+++ b/plugin_tests/elasticsearch_test.py
@@ -69,12 +69,12 @@ def testCreateElasticsearchSource(self):
         username = ''
         password = ''
         baseURL = 'http://elasticsearch.com'
-        table = 'mytable'
+        index = 'myindex'
         params = {
             'name': name,
             'username': username,
             'password': password,
-            'table': table,
+            'index': index,
             'baseURL': baseURL
         }
         response = self.request(path=path, method='POST', params=params, user=self._user)
@@ -85,4 +85,4 @@ def testCreateElasticsearchSource(self):
         self.assertEquals(elasticsearchSource['name'], name, 'incorrect elasticsearch source name')
         self.assertEquals(minerva_metadata['source_type'], 'elasticsearch', 'incorrect elasticsearch source type')
         self.assertEquals(minerva_metadata['elasticsearch_params']['base_url'], baseURL, 'incorrect elasticsearch source baseURL')
-        self.assertEquals(minerva_metadata['elasticsearch_params']['table'], table, 'incorrect elasticsearch source table')
+        self.assertEquals(minerva_metadata['elasticsearch_params']['index'], index, 'incorrect elasticsearch source index')
diff --git a/server/rest/elasticsearch_source.py b/server/rest/elasticsearch_source.py
index d2b62b6d..0735f312 100644
--- a/server/rest/elasticsearch_source.py
+++ b/server/rest/elasticsearch_source.py
@@ -37,13 +37,13 @@ def createElasticsearchSource(self, params):
         baseURL = params['baseURL']
         parsedUrl = getUrlParts(baseURL)
         hostName = parsedUrl.netloc
-        table = params['table']
+        index = params['index']
         username = params['username'] if 'username' in params else None
         password = params['password'] if 'password' in params else None
         minerva_metadata = {
             'source_type': 'elasticsearch',
             'elasticsearch_params': {
-                'table': table,
+                'index': index,
                 'base_url': baseURL,
                 'host_name': hostName
             }
@@ -59,7 +59,7 @@ def createElasticsearchSource(self, params):
         .responseClass('Item')
         .param('name', 'The name of the elasticsearch source', required=True)
         .param('baseURL', 'URL of the elasticsearch service', required=True)
-        .param('table', 'Table of interest', required=True)
+        .param('index', 'Index of interest', required=True)
         .param('username', 'Elasticsearch username', required=False)
         .param('password', 'Elasticsearch password', required=False)
         .errorResponse('Write permission denied on the source folder.', 403))
diff --git a/web_external/js/views/widgets/AddElasticsearchSourceWidget.js b/web_external/js/views/widgets/AddElasticsearchSourceWidget.js
index 496c3cd4..fac21d07 100644
--- a/web_external/js/views/widgets/AddElasticsearchSourceWidget.js
+++ b/web_external/js/views/widgets/AddElasticsearchSourceWidget.js
@@ -9,7 +9,7 @@ minerva.views.AddElasticsearchSourceWidget = minerva.View.extend({
             var params = {
                 name:     this.$('#m-elasticsearch-name').val(),
                 baseURL:  this.$('#m-elasticsearch-uri').val(),
-                table:  this.$('#m-elasticsearch-table').val(),
+                index:  this.$('#m-elasticsearch-index').val(),
                 username: this.$('#m-elasticsearch-username').val(),
                 password: this.$('#m-elasticsearch-password').val()
             };
diff --git a/web_external/templates/widgets/addElasticsearchSourceWidget.jade b/web_external/templates/widgets/addElasticsearchSourceWidget.jade
index 48ed6be6..e544858d 100644
--- a/web_external/templates/widgets/addElasticsearchSourceWidget.jade
+++ b/web_external/templates/widgets/addElasticsearchSourceWidget.jade
@@ -14,8 +14,8 @@
           label.control-label(for="m-elasticsearch-uri") Elasticsearch baseURL
           input.input-sm#m-elasticsearch-uri.form-control(type="text", placeholder="http://elasticsearch.com")
         .form-group
-          label.control-label(for="m-elasticsearch-table") Elasticsearch table
-          input.input-sm#m-elasticsearch-table.form-control(type="text", placeholder="table")
+          label.control-label(for="m-elasticsearch-index") Elasticsearch index
+          input.input-sm#m-elasticsearch-index.form-control(type="text", placeholder="index")
         .form-group
           label.control-label(for="m-elasticsearch-username") Username
           input.input-sm#m-elasticsearch-username.form-control(type="text", placeholder="username")

From 25c7dac91b9bf77818c5f83ad9f591194ea1f642 Mon Sep 17 00:00:00 2001
From: Dan LaManna <dan.lamanna@kitware.com>
Date: Mon, 12 Oct 2015 09:29:28 -0400
Subject: [PATCH 14/20] Add Elasticsearch Dataset Endpoint

---
 plugin_tests/elasticsearch_test.py   | 29 +++++++++++-
 server/loader.py                     |  4 +-
 server/rest/elasticsearch_dataset.py | 70 ++++++++++++++++++++++++++++
 3 files changed, 101 insertions(+), 2 deletions(-)
 create mode 100644 server/rest/elasticsearch_dataset.py

diff --git a/plugin_tests/elasticsearch_test.py b/plugin_tests/elasticsearch_test.py
index 52bef909..3a2ecc2e 100644
--- a/plugin_tests/elasticsearch_test.py
+++ b/plugin_tests/elasticsearch_test.py
@@ -61,7 +61,7 @@ def setUp(self):
 
     def testCreateElasticsearchSource(self):
         """
-        Test the minerva ELASTICSEARCH source API endpoints.
+        Test the minerva Elasticsearch source API endpoints.
         """
 
         path = '/minerva_source_elasticsearch'
@@ -86,3 +86,30 @@ def testCreateElasticsearchSource(self):
         self.assertEquals(minerva_metadata['source_type'], 'elasticsearch', 'incorrect elasticsearch source type')
         self.assertEquals(minerva_metadata['elasticsearch_params']['base_url'], baseURL, 'incorrect elasticsearch source baseURL')
         self.assertEquals(minerva_metadata['elasticsearch_params']['index'], index, 'incorrect elasticsearch source index')
+
+        return elasticsearchSource
+
+    def testCreateElasticsearchDataset(self):
+        """
+        Test the minerva Elasticsearch dataset API endpoints.
+        """
+        elasticsearchSource = self.testCreateElasticsearchSource()
+
+        name = 'testElasticsearch'
+        datasetName = 'testElasticsearchDataset'
+        params = {
+            'name': name,
+            'elasticsearchSourceId': elasticsearchSource['_id'],
+            'datasetName': datasetName
+        }
+
+        response = self.request(path='/minerva_dataset_elasticsearch',
+                                method='POST',
+                                params=params,
+                                user=self._user)
+        self.assertStatusOk(response)
+        elasticsearchDataset = response.json
+        minerva_metadata = elasticsearchDataset['meta']['minerva']
+        self.assertEquals(elasticsearchDataset['name'], name, 'incorrect elasticsearch dataset name')
+        self.assertEquals(minerva_metadata['source_id'], elasticsearchSource['_id'], 'incorrect elasticsearch source_id')
+        self.assertEquals(minerva_metadata['datasetName'], datasetName, 'incorrect elasticsearch datasetName')
diff --git a/server/loader.py b/server/loader.py
index 51fb1cf5..9d7d8de5 100644
--- a/server/loader.py
+++ b/server/loader.py
@@ -28,7 +28,7 @@
 
 from girder.plugins.minerva.rest import \
         analysis, dataset, s3_dataset, session, shapefile, geocode, source, \
-        wms_dataset, wms_source, geojson_dataset, elasticsearch_source
+        wms_dataset, wms_source, geojson_dataset, elasticsearch_source, elasticsearch_dataset
 from girder.plugins.minerva.utility.minerva_utility import decryptCredentials
 
 
@@ -196,5 +196,7 @@ def load(info):
 
     info['apiRoot'].minerva_source_elasticsearch = \
         elasticsearch_source.ElasticsearchSource()
+    info['apiRoot'].minerva_dataset_elasticsearch = \
+        elasticsearch_dataset.ElasticsearchDataset()
 
     info['serverRoot'].wms_proxy = WmsProxy()
diff --git a/server/rest/elasticsearch_dataset.py b/server/rest/elasticsearch_dataset.py
new file mode 100644
index 00000000..4529c5a5
--- /dev/null
+++ b/server/rest/elasticsearch_dataset.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+from base64 import b64encode
+from girder.api import access
+from girder.api.describe import Description
+from girder.api.rest import loadmodel
+from girder.api.rest import getUrlParts
+from girder.constants import AccessType
+
+from girder.plugins.minerva.rest.dataset import Dataset
+from girder.plugins.minerva.utility.minerva_utility import decryptCredentials
+
+
+class ElasticsearchDataset(Dataset):
+
+    def __init__(self):
+        self.resourceName = 'minerva_dataset_elasticsearch'
+        self.route('POST', (), self.createElasticsearchDataset)
+
+    @access.user
+    @loadmodel(map={'elasticsearchSourceId': 'elasticsearchSource'}, model='item',
+               level=AccessType.READ)
+    def createElasticsearchDataset(self, elasticsearchSource, params):
+        baseURL = elasticsearchSource['meta']['minerva']['elasticsearch_params']['base_url']
+        parsedUrl = getUrlParts(baseURL)
+
+        if 'credentials' in elasticsearchSource['meta']['minerva']['elasticsearch_params']:
+            credentials = (
+                elasticsearchSource['meta']['minerva']['elasticsearch_params']['credentials']
+            )
+            basic_auth = 'Basic ' + b64encode(decryptCredentials(credentials))
+            headers = {'Authorization': basic_auth}
+        else:
+            headers = {}
+            credentials = None
+
+        self.requireParams(('name'), params)
+        name = params['name']
+        minerva_metadata = {
+            'dataset_type': 'elasticsearch',
+            'source_id': elasticsearchSource['_id'],
+            'base_url': baseURL
+        }
+        if credentials:
+            minerva_metadata['credentials'] = credentials
+        dataset = self.constructDataset(name, minerva_metadata)
+        return dataset
+    createElasticsearchDataset.description = (
+        Description('Create an Elasticsearch Dataset from an Elasticsearch Source.')
+        .responseClass('Item')
+        .param('name', 'The name of the Elasticsearch dataset', required=True)
+        .param('elasticsearchSourceId', 'Item ID of the Elasticsearch Source', required=True)
+        .errorResponse('ID was invalid.')
+        .errorResponse('Read permission denied on the Item.', 403))

From e3789f5fb5a1ec244360b9a54ba46ffa8e8b920d Mon Sep 17 00:00:00 2001
From: Dan LaManna <dan.lamanna@kitware.com>
Date: Mon, 12 Oct 2015 12:49:29 -0400
Subject: [PATCH 15/20] Add elasticsearch query endpoint

---
 server/jobs/elasticsearch_worker.py           | 113 ++++++++++++++++++
 server/loader.py                              |   2 +
 server/rest/elasticsearch_source.py           |  67 ++++++++++-
 .../js/views/widgets/ElasticsearchWidget.js   |   8 +-
 4 files changed, 186 insertions(+), 4 deletions(-)
 create mode 100644 server/jobs/elasticsearch_worker.py

diff --git a/server/jobs/elasticsearch_worker.py b/server/jobs/elasticsearch_worker.py
new file mode 100644
index 00000000..8e2c3954
--- /dev/null
+++ b/server/jobs/elasticsearch_worker.py
@@ -0,0 +1,113 @@
+import json
+import os
+import shutil
+import sys
+import tempfile
+import traceback
+
+from elasticsearch import Elasticsearch
+
+from girder.constants import AccessType
+from girder.utility import config
+from girder.utility.model_importer import ModelImporter
+from girder.plugins.jobs.constants import JobStatus
+from girder.plugins.minerva.utility.dataset_utility import \
+    jsonArrayHead
+
+from girder.plugins.minerva.utility.minerva_utility import decryptCredentials
+
+import girder_client
+
+
+def run(job):
+    job_model = ModelImporter.model('job', 'jobs')
+    job_model.updateJob(job, status=JobStatus.RUNNING)
+
+    try:
+        kwargs = job['kwargs']
+        # TODO better to create a job token rather than a user token?
+        token = kwargs['token']
+        datasetId = str(kwargs['dataset']['_id'])
+
+        # connect to girder and upload the file
+        # TODO will probably have to change this from local to romanesco
+        # so that can work on worker machine
+        # at least need host connection info
+        girderPort = config.getConfig()['server.socket_port']
+        client = girder_client.GirderClient(port=girderPort)
+        client.token = token['_id']
+
+        # Get datasource
+        source = client.getItem(kwargs['params']['sourceId'])
+        esUrl = 'https://%s@%s' % (decryptCredentials(
+            source['meta']['minerva']['elasticsearch_params']['credentials']),
+                                   source['meta']['minerva']['elasticsearch_params']['host_name'])
+        es = Elasticsearch([esUrl])
+
+
+        # TODO sleeping in async thread, probably starving other tasks
+        # would be better to split this into two or more parts, creating
+        # additional jobs as needed
+        searchResult = es.search(
+            index=source['meta']['minerva']['elasticsearch_params']['index'],
+            body=json.loads(kwargs['params']['searchParams']))
+
+        # write the output to a json file
+        tmpdir = tempfile.mkdtemp()
+        outFilepath = tempfile.mkstemp(suffix='.json', dir=tmpdir)[1]
+        writer = open(outFilepath, 'w')
+        writer.write(json.dumps(searchResult))
+        writer.close()
+
+        # rename the file so it will have the right name when uploaded
+        # could probably be done post upload
+        outFilename = 'search.json'
+        humanFilepath = os.path.join(tmpdir, outFilename)
+        shutil.move(outFilepath, humanFilepath)
+
+
+        client.uploadFileToItem(datasetId, humanFilepath)
+
+        # TODO some stuff here using models will only work on a local job
+        # will have to be rewritten using girder client to work in romanesco
+        # non-locally
+
+        user_model = ModelImporter.model('user')
+        user = user_model.load(job['userId'], force=True)
+        item_model = ModelImporter.model('item')
+        # TODO only works locally
+        dataset = item_model.load(datasetId, level=AccessType.WRITE, user=user)
+        metadata = dataset['meta']
+        minerva_metadata = metadata['minerva']
+
+        # TODO only works locally
+        file_model = ModelImporter.model('file')
+        existing = file_model.findOne({
+            'itemId': dataset['_id'],
+            'name': outFilename
+        })
+        if existing:
+            minerva_metadata['original_files'] = [{
+                '_id': existing['_id'],
+                'name': outFilename
+            }]
+        else:
+            raise (Exception('Cannot find file %s in dataset %s' %
+                   (outFilename, datasetId)))
+
+        jsonRow = jsonArrayHead(humanFilepath, limit=1)[0]
+        minerva_metadata['json_row'] = jsonRow
+
+        shutil.rmtree(tmpdir)
+
+        metadata['minerva'] = minerva_metadata
+        # TODO only works locally
+        item_model.setMetadata(dataset, metadata)
+        # TODO only works locally
+        job_model.updateJob(job, status=JobStatus.SUCCESS)
+    except Exception:
+        t, val, tb = sys.exc_info()
+        log = '%s: %s\n%s' % (t.__name__, repr(val), traceback.extract_tb(tb))
+        # TODO only works locally
+        job_model.updateJob(job, status=JobStatus.ERROR, log=log)
+        raise
diff --git a/server/loader.py b/server/loader.py
index 9d7d8de5..9c555e59 100644
--- a/server/loader.py
+++ b/server/loader.py
@@ -196,6 +196,8 @@ def load(info):
 
     info['apiRoot'].minerva_source_elasticsearch = \
         elasticsearch_source.ElasticsearchSource()
+    info['apiRoot'].minerva_query_elasticsearch = \
+        elasticsearch_source.ElasticsearchQuery()
     info['apiRoot'].minerva_dataset_elasticsearch = \
         elasticsearch_dataset.ElasticsearchDataset()
 
diff --git a/server/rest/elasticsearch_source.py b/server/rest/elasticsearch_source.py
index 0735f312..7ea5a298 100644
--- a/server/rest/elasticsearch_source.py
+++ b/server/rest/elasticsearch_source.py
@@ -19,11 +19,15 @@
 
 from girder.api import access
 from girder.api.describe import Description
-from girder.api.rest import getUrlParts
+from girder.api.rest import getUrlParts, Resource
 
 from girder.plugins.minerva.rest.source import Source
 from girder.plugins.minerva.utility.minerva_utility import encryptCredentials
 
+from girder.plugins.minerva.utility.minerva_utility import (findAnalysisFolder,
+                                                            findAnalysisByName,
+                                                            findDatasetFolder)
+
 
 class ElasticsearchSource(Source):
 
@@ -63,3 +67,64 @@ def createElasticsearchSource(self, params):
         .param('username', 'Elasticsearch username', required=False)
         .param('password', 'Elasticsearch password', required=False)
         .errorResponse('Write permission denied on the source folder.', 403))
+
+
+class ElasticsearchQuery(Resource):
+
+    def __init__(self):
+        self.resourceName = 'minerva_query_elasticsearch'
+        self.route('POST', (), self.queryElasticsearch)
+
+    @access.user
+    def queryElasticsearch(self, params):
+        currentUser = self.getCurrentUser()
+        datasetName = params['datasetName']
+        elasticsearchParams = params['searchParams']
+
+        datasetFolder = findDatasetFolder(currentUser, currentUser)
+        dataset = (self.model('item').createItem(datasetName,
+                                                 currentUser,
+                                                 datasetFolder,
+                                                 'created by elasticsearch query'))
+
+        user, token = self.getCurrentUser(returnToken=True)
+        kwargs = {
+            'params': params,
+            'user': currentUser,
+            'dataset': dataset,
+            'token': token,
+            'sourceId': params['sourceId']
+        }
+
+        job = self.model('job', 'jobs').createLocalJob(
+            title='elasticsearch: %s' % datasetName,
+            user=currentUser,
+            type='elasticsearch',
+            public=False,
+            kwargs=kwargs,
+            module='girder.plugins.minerva.jobs.elasticsearch_worker',
+            async=True)
+
+        if 'meta' in dataset:
+            metadata = dataset['meta']
+        else:
+            metadata = {}
+
+        minerva_metadata = {
+            'dataset_id': dataset['_id'],
+            'source': 'elasticsearch',
+            'elasticsearch_params': elasticsearchParams,
+            'original_type': 'json'
+        }
+        metadata['minerva'] = minerva_metadata
+        self.model('item').setMetadata(dataset, metadata)
+
+        self.model('job', 'jobs').scheduleJob(job)
+
+        return minerva_metadata
+
+    queryElasticsearch.description = (
+        Description('Query an elasticsearch source.')
+        .param('sourceId', 'Item id of the elasticsearch source to query')
+        .param('datasetName', 'The name of the resulting dataset')
+        .param('query', 'JSON Body of an elasticsearch query'))
diff --git a/web_external/js/views/widgets/ElasticsearchWidget.js b/web_external/js/views/widgets/ElasticsearchWidget.js
index 529e782f..ac82f0c3 100644
--- a/web_external/js/views/widgets/ElasticsearchWidget.js
+++ b/web_external/js/views/widgets/ElasticsearchWidget.js
@@ -29,11 +29,12 @@ minerva.views.ElasticsearchWidget = minerva.View.extend({
 
             var data = {
                 datasetName: datasetName,
-                searchParams: searchParams
+                searchParams: searchParams,
+                sourceId: this.source.get('id')
             };
 
             girder.restRequest({
-                path: 'minerva_elasticsearch_query',
+                path: 'minerva_query_elasticsearch',
                 type: 'POST',
                 data: data
             }).done(_.bind(function () {
@@ -44,7 +45,8 @@ minerva.views.ElasticsearchWidget = minerva.View.extend({
         }
     },
 
-    initialize: function () {
+    initialize: function (settings) {
+        this.source = settings.source;
     },
 
     render: function () {

From 5174e52d406e0c4b2e1a52bde80a893d811eaec3 Mon Sep 17 00:00:00 2001
From: Dan LaManna <dan.lamanna@kitware.com>
Date: Mon, 12 Oct 2015 13:23:29 -0400
Subject: [PATCH 16/20] Fix style errors

---
 server/rest/elasticsearch_dataset.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/server/rest/elasticsearch_dataset.py b/server/rest/elasticsearch_dataset.py
index 4529c5a5..3625eeb4 100644
--- a/server/rest/elasticsearch_dataset.py
+++ b/server/rest/elasticsearch_dataset.py
@@ -34,26 +34,27 @@ def __init__(self):
         self.route('POST', (), self.createElasticsearchDataset)
 
     @access.user
-    @loadmodel(map={'elasticsearchSourceId': 'elasticsearchSource'}, model='item',
-               level=AccessType.READ)
+    @loadmodel(map={'elasticsearchSourceId': 'elasticsearchSource'},
+               model='item', level=AccessType.READ)
     def createElasticsearchDataset(self, elasticsearchSource, params):
-        baseURL = elasticsearchSource['meta']['minerva']['elasticsearch_params']['base_url']
+        minerva_meta = elasticsearchSource['meta']['minerva']
+        baseURL = minerva_meta['elasticsearch_params']['base_url']
         parsedUrl = getUrlParts(baseURL)
 
-        if 'credentials' in elasticsearchSource['meta']['minerva']['elasticsearch_params']:
+        if 'credentials' in minerva_meta['elasticsearch_params']:
             credentials = (
-                elasticsearchSource['meta']['minerva']['elasticsearch_params']['credentials']
+                minerva_meta['elasticsearch_params']['credentials']
             )
             basic_auth = 'Basic ' + b64encode(decryptCredentials(credentials))
             headers = {'Authorization': basic_auth}
         else:
-            headers = {}
             credentials = None
 
         self.requireParams(('name'), params)
         name = params['name']
         minerva_metadata = {
             'dataset_type': 'elasticsearch',
+            'datasetName': params['datasetName'],
             'source_id': elasticsearchSource['_id'],
             'base_url': baseURL
         }
@@ -62,9 +63,11 @@ def createElasticsearchDataset(self, elasticsearchSource, params):
         dataset = self.constructDataset(name, minerva_metadata)
         return dataset
     createElasticsearchDataset.description = (
-        Description('Create an Elasticsearch Dataset from an Elasticsearch Source.')
+        Description(('Create an Elasticsearch Dataset from '
+                     'an Elasticsearch Source.'))
         .responseClass('Item')
         .param('name', 'The name of the Elasticsearch dataset', required=True)
-        .param('elasticsearchSourceId', 'Item ID of the Elasticsearch Source', required=True)
+        .param('elasticsearchSourceId',
+               'Item ID of the Elasticsearch Source', required=True)
         .errorResponse('ID was invalid.')
         .errorResponse('Read permission denied on the Item.', 403))

From be45937d2e0bddd08a90c463683c7909a8689580 Mon Sep 17 00:00:00 2001
From: Dan LaManna <dan.lamanna@kitware.com>
Date: Mon, 12 Oct 2015 13:36:05 -0400
Subject: [PATCH 17/20] Fix flake8 errors

---
 server/rest/elasticsearch_dataset.py |  3 ---
 server/rest/elasticsearch_source.py  | 17 ++++++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/server/rest/elasticsearch_dataset.py b/server/rest/elasticsearch_dataset.py
index 3625eeb4..bb327dde 100644
--- a/server/rest/elasticsearch_dataset.py
+++ b/server/rest/elasticsearch_dataset.py
@@ -39,14 +39,11 @@ def __init__(self):
     def createElasticsearchDataset(self, elasticsearchSource, params):
         minerva_meta = elasticsearchSource['meta']['minerva']
         baseURL = minerva_meta['elasticsearch_params']['base_url']
-        parsedUrl = getUrlParts(baseURL)
 
         if 'credentials' in minerva_meta['elasticsearch_params']:
             credentials = (
                 minerva_meta['elasticsearch_params']['credentials']
             )
-            basic_auth = 'Basic ' + b64encode(decryptCredentials(credentials))
-            headers = {'Authorization': basic_auth}
         else:
             credentials = None
 
diff --git a/server/rest/elasticsearch_source.py b/server/rest/elasticsearch_source.py
index 7ea5a298..90df7b96 100644
--- a/server/rest/elasticsearch_source.py
+++ b/server/rest/elasticsearch_source.py
@@ -24,9 +24,7 @@
 from girder.plugins.minerva.rest.source import Source
 from girder.plugins.minerva.utility.minerva_utility import encryptCredentials
 
-from girder.plugins.minerva.utility.minerva_utility import (findAnalysisFolder,
-                                                            findAnalysisByName,
-                                                            findDatasetFolder)
+from girder.plugins.minerva.utility.minerva_utility import findDatasetFolder
 
 
 class ElasticsearchSource(Source):
@@ -77,15 +75,20 @@ def __init__(self):
 
     @access.user
     def queryElasticsearch(self, params):
+        """
+        Creates a dataset to store the results for an elastic search query,
+        then calls a local job running the elasticsearch_worker.
+        """
         currentUser = self.getCurrentUser()
         datasetName = params['datasetName']
         elasticsearchParams = params['searchParams']
 
         datasetFolder = findDatasetFolder(currentUser, currentUser)
-        dataset = (self.model('item').createItem(datasetName,
-                                                 currentUser,
-                                                 datasetFolder,
-                                                 'created by elasticsearch query'))
+        dataset = (self.model('item').createItem(
+            datasetName,
+            currentUser,
+            datasetFolder,
+            'created by elasticsearch query'))
 
         user, token = self.getCurrentUser(returnToken=True)
         kwargs = {

From 8f729aef53bbebbeafeeb37e657e4b64e027e94f Mon Sep 17 00:00:00 2001
From: Dan LaManna <dan.lamanna@kitware.com>
Date: Mon, 12 Oct 2015 13:48:45 -0400
Subject: [PATCH 18/20] Fix flake8 errors

---
 server/jobs/elasticsearch_worker.py  | 4 +---
 server/rest/elasticsearch_dataset.py | 7 +------
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/server/jobs/elasticsearch_worker.py b/server/jobs/elasticsearch_worker.py
index 8e2c3954..09bd096f 100644
--- a/server/jobs/elasticsearch_worker.py
+++ b/server/jobs/elasticsearch_worker.py
@@ -41,10 +41,9 @@ def run(job):
         source = client.getItem(kwargs['params']['sourceId'])
         esUrl = 'https://%s@%s' % (decryptCredentials(
             source['meta']['minerva']['elasticsearch_params']['credentials']),
-                                   source['meta']['minerva']['elasticsearch_params']['host_name'])
+            source['meta']['minerva']['elasticsearch_params']['host_name'])
         es = Elasticsearch([esUrl])
 
-
         # TODO sleeping in async thread, probably starving other tasks
         # would be better to split this into two or more parts, creating
         # additional jobs as needed
@@ -65,7 +64,6 @@ def run(job):
         humanFilepath = os.path.join(tmpdir, outFilename)
         shutil.move(outFilepath, humanFilepath)
 
-
         client.uploadFileToItem(datasetId, humanFilepath)
 
         # TODO some stuff here using models will only work on a local job
diff --git a/server/rest/elasticsearch_dataset.py b/server/rest/elasticsearch_dataset.py
index bb327dde..f4b7defb 100644
--- a/server/rest/elasticsearch_dataset.py
+++ b/server/rest/elasticsearch_dataset.py
@@ -16,15 +16,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 ###############################################################################
-from base64 import b64encode
 from girder.api import access
 from girder.api.describe import Description
 from girder.api.rest import loadmodel
-from girder.api.rest import getUrlParts
 from girder.constants import AccessType
 
 from girder.plugins.minerva.rest.dataset import Dataset
-from girder.plugins.minerva.utility.minerva_utility import decryptCredentials
 
 
 class ElasticsearchDataset(Dataset):
@@ -41,9 +38,7 @@ def createElasticsearchDataset(self, elasticsearchSource, params):
         baseURL = minerva_meta['elasticsearch_params']['base_url']
 
         if 'credentials' in minerva_meta['elasticsearch_params']:
-            credentials = (
-                minerva_meta['elasticsearch_params']['credentials']
-            )
+            credentials = (minerva_meta['elasticsearch_params']['credentials'])
         else:
             credentials = None
 

From 2ffbe00b2f909c78defd632bf110084fe8fa8d17 Mon Sep 17 00:00:00 2001
From: Dan LaManna <dan.lamanna@kitware.com>
Date: Mon, 12 Oct 2015 14:57:34 -0400
Subject: [PATCH 19/20] Remove ES dataset endpoint

---
 plugin_tests/elasticsearch_test.py   | 25 -----------
 server/loader.py                     |  4 +-
 server/rest/elasticsearch_dataset.py | 65 ----------------------------
 3 files changed, 1 insertion(+), 93 deletions(-)
 delete mode 100644 server/rest/elasticsearch_dataset.py

diff --git a/plugin_tests/elasticsearch_test.py b/plugin_tests/elasticsearch_test.py
index 3a2ecc2e..e5d09d01 100644
--- a/plugin_tests/elasticsearch_test.py
+++ b/plugin_tests/elasticsearch_test.py
@@ -88,28 +88,3 @@ def testCreateElasticsearchSource(self):
         self.assertEquals(minerva_metadata['elasticsearch_params']['index'], index, 'incorrect elasticsearch source index')
 
         return elasticsearchSource
-
-    def testCreateElasticsearchDataset(self):
-        """
-        Test the minerva Elasticsearch dataset API endpoints.
-        """
-        elasticsearchSource = self.testCreateElasticsearchSource()
-
-        name = 'testElasticsearch'
-        datasetName = 'testElasticsearchDataset'
-        params = {
-            'name': name,
-            'elasticsearchSourceId': elasticsearchSource['_id'],
-            'datasetName': datasetName
-        }
-
-        response = self.request(path='/minerva_dataset_elasticsearch',
-                                method='POST',
-                                params=params,
-                                user=self._user)
-        self.assertStatusOk(response)
-        elasticsearchDataset = response.json
-        minerva_metadata = elasticsearchDataset['meta']['minerva']
-        self.assertEquals(elasticsearchDataset['name'], name, 'incorrect elasticsearch dataset name')
-        self.assertEquals(minerva_metadata['source_id'], elasticsearchSource['_id'], 'incorrect elasticsearch source_id')
-        self.assertEquals(minerva_metadata['datasetName'], datasetName, 'incorrect elasticsearch datasetName')
diff --git a/server/loader.py b/server/loader.py
index 9c555e59..f2521e0f 100644
--- a/server/loader.py
+++ b/server/loader.py
@@ -28,7 +28,7 @@
 
 from girder.plugins.minerva.rest import \
         analysis, dataset, s3_dataset, session, shapefile, geocode, source, \
-        wms_dataset, wms_source, geojson_dataset, elasticsearch_source, elasticsearch_dataset
+        wms_dataset, wms_source, geojson_dataset, elasticsearch_source
 from girder.plugins.minerva.utility.minerva_utility import decryptCredentials
 
 
@@ -198,7 +198,5 @@ def load(info):
         elasticsearch_source.ElasticsearchSource()
     info['apiRoot'].minerva_query_elasticsearch = \
         elasticsearch_source.ElasticsearchQuery()
-    info['apiRoot'].minerva_dataset_elasticsearch = \
-        elasticsearch_dataset.ElasticsearchDataset()
 
     info['serverRoot'].wms_proxy = WmsProxy()
diff --git a/server/rest/elasticsearch_dataset.py b/server/rest/elasticsearch_dataset.py
deleted file mode 100644
index f4b7defb..00000000
--- a/server/rest/elasticsearch_dataset.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-###############################################################################
-#  Copyright Kitware Inc.
-#
-#  Licensed under the Apache License, Version 2.0 ( the "License" );
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-###############################################################################
-from girder.api import access
-from girder.api.describe import Description
-from girder.api.rest import loadmodel
-from girder.constants import AccessType
-
-from girder.plugins.minerva.rest.dataset import Dataset
-
-
-class ElasticsearchDataset(Dataset):
-
-    def __init__(self):
-        self.resourceName = 'minerva_dataset_elasticsearch'
-        self.route('POST', (), self.createElasticsearchDataset)
-
-    @access.user
-    @loadmodel(map={'elasticsearchSourceId': 'elasticsearchSource'},
-               model='item', level=AccessType.READ)
-    def createElasticsearchDataset(self, elasticsearchSource, params):
-        minerva_meta = elasticsearchSource['meta']['minerva']
-        baseURL = minerva_meta['elasticsearch_params']['base_url']
-
-        if 'credentials' in minerva_meta['elasticsearch_params']:
-            credentials = (minerva_meta['elasticsearch_params']['credentials'])
-        else:
-            credentials = None
-
-        self.requireParams(('name'), params)
-        name = params['name']
-        minerva_metadata = {
-            'dataset_type': 'elasticsearch',
-            'datasetName': params['datasetName'],
-            'source_id': elasticsearchSource['_id'],
-            'base_url': baseURL
-        }
-        if credentials:
-            minerva_metadata['credentials'] = credentials
-        dataset = self.constructDataset(name, minerva_metadata)
-        return dataset
-    createElasticsearchDataset.description = (
-        Description(('Create an Elasticsearch Dataset from '
-                     'an Elasticsearch Source.'))
-        .responseClass('Item')
-        .param('name', 'The name of the Elasticsearch dataset', required=True)
-        .param('elasticsearchSourceId',
-               'Item ID of the Elasticsearch Source', required=True)
-        .errorResponse('ID was invalid.')
-        .errorResponse('Read permission denied on the Item.', 403))

From dcfc226dd5febcacb62a241725749f0ea5bea0e1 Mon Sep 17 00:00:00 2001
From: Dan LaManna <dan.lamanna@kitware.com>
Date: Tue, 13 Oct 2015 12:51:24 -0400
Subject: [PATCH 20/20] Using updateMinervaMetadata built-in

---
 server/rest/elasticsearch_source.py | 25 +++++++++----------------
 1 file changed, 9 insertions(+), 16 deletions(-)

diff --git a/server/rest/elasticsearch_source.py b/server/rest/elasticsearch_source.py
index 90df7b96..303ed239 100644
--- a/server/rest/elasticsearch_source.py
+++ b/server/rest/elasticsearch_source.py
@@ -22,9 +22,8 @@
 from girder.api.rest import getUrlParts, Resource
 
 from girder.plugins.minerva.rest.source import Source
-from girder.plugins.minerva.utility.minerva_utility import encryptCredentials
-
-from girder.plugins.minerva.utility.minerva_utility import findDatasetFolder
+from girder.plugins.minerva.utility.minerva_utility import \
+    encryptCredentials, findDatasetFolder, updateMinervaMetadata
 
 
 class ElasticsearchSource(Source):
@@ -76,8 +75,8 @@ def __init__(self):
     @access.user
     def queryElasticsearch(self, params):
         """
-        Creates a dataset to store the results for an elastic search query,
-        then calls a local job running the elasticsearch_worker.
+        Creates a local job to run the elasticsearch_worker, the job will store
+        the results of the elastic search query in a dataset.
         """
         currentUser = self.getCurrentUser()
         datasetName = params['datasetName']
@@ -108,23 +107,17 @@ def queryElasticsearch(self, params):
             module='girder.plugins.minerva.jobs.elasticsearch_worker',
             async=True)
 
-        if 'meta' in dataset:
-            metadata = dataset['meta']
-        else:
-            metadata = {}
-
         minerva_metadata = {
-            'dataset_id': dataset['_id'],
+            'dataset_type': 'json',
+            'source_id': params['sourceId'],
             'source': 'elasticsearch',
-            'elasticsearch_params': elasticsearchParams,
-            'original_type': 'json'
+            'elasticsearch_params': elasticsearchParams
         }
-        metadata['minerva'] = minerva_metadata
-        self.model('item').setMetadata(dataset, metadata)
+        updateMinervaMetadata(dataset, minerva_metadata)
 
         self.model('job', 'jobs').scheduleJob(job)
 
-        return minerva_metadata
+        return job
 
     queryElasticsearch.description = (
         Description('Query an elasticsearch source.')