Merge pull request #72 from akintolga/master

feat(projections) - Source projections

eve_elastic/elastic.py

@@ -2,6 +2,8 @@
 import ast
 import json
 import arrow
+import ciso8601
+import pytz  # NOQA
 import logging
 import elasticsearch
 
@@ -25,10 +27,12 @@ def parse_date(date_str):
         return None
 
     try:
-        date = arrow.get(date_str)
+        date = ciso8601.parse_datetime(date_str)
+        if not date:
+            date = arrow.get(date_str).datetime
     except TypeError:
-        date = arrow.get(date_str[0])
-    return date.datetime
+        date = arrow.get(date_str[0]).datetime
+    return date
 
 
 def get_dates(schema):
@@ -416,7 +420,11 @@ def find(self, resource, req, sub_resource_lookup):
         if 'es_highlight' in source_config and self.should_highlight(req):
             query['highlight'] = source_config['es_highlight']
 
-        args = self._es_args(resource)
+        source_projections = None
+        if self.should_project(req):
+            source_projections = self.get_projected_fields(req)
+
+        args = self._es_args(resource, source_projections=source_projections)
         try:
             hits = self.elastic(resource).search(body=query, **args)
         except elasticsearch.exceptions.RequestError as e:
@@ -442,7 +450,7 @@ def should_aggregate(self, req):
 
     def should_highlight(self, req):
         """
-        Check the given argument parameter to decide if aggregations needed.
+        Check the given argument parameter to decide if highlights needed.
 
         argument value is expected to be '0' or '1'
         """
@@ -451,6 +459,28 @@ def should_highlight(self, req):
         except:
             return False
 
+    def should_project(self, req):
+        """
+        Check the given argument parameter to decide if projections needed.
+
+        argument value is expected to be a list of strings
+        """
+        try:
+            return req.args and json.loads(req.args.get('projections', []))
+        except:
+            return False
+
+    def get_projected_fields(self, req):
+        """
+        Returns the projected fields from request.
+
+        """
+        try:
+            args = getattr(req, 'args', {})
+            return ','.join(json.loads(args.get('projections')))
+        except:
+            return None
+
     def find_one(self, resource, req, **lookup):
         """Find single document, if there is _id in lookup use that, otherwise filter."""
 
@@ -586,13 +616,15 @@ def _parse_hits(self, hits, resource):
             docs.append(format_doc(hit, schema, dates))
         return ElasticCursor(hits, docs)
 
-    def _es_args(self, resource, refresh=None):
+    def _es_args(self, resource, refresh=None, source_projections=None):
         """Get index and doctype args."""
         datasource = self.get_datasource(resource)
         args = {
             'index': self._resource_index(resource),
             'doc_type': datasource[0],
         }
+        if source_projections:
+            args['_source'] = source_projections
         if refresh:
             args['refresh'] = refresh
         return args

setup.py

@@ -25,6 +25,8 @@
     tests_require=['nose', 'flake8'],
     install_requires=[
         'arrow>=0.4.2',
+        'ciso8601>=1.0.2',
+        'pytz>=2015.4',
         'elasticsearch>=1.2.0,<2.0.0',
         'Eve>=0.4',
     ],

test/test_elastic.py

@@ -308,6 +308,20 @@ def test_search_via_source_param_and_without_highlight(self):
             es_highlight = res[0].get('es_highlight')
             self.assertIsNone(es_highlight)
 
+    def test_search_via_source_param_and_with_source_projection(self):
+        query = {'query': {'query_string': {'query': 'foo'}}}
+        with self.app.app_context():
+            self.app.data.insert('items_with_description', [{'uri': 'foo',
+                                                             'description': 'This is foo',
+                                                             'name': 'foo'}])
+            self.app.data.insert('items_with_description', [{'uri': 'bar', 'name': 'bar'}])
+            req = ParsedRequest()
+            req.args = {'source': json.dumps(query), 'projections': json.dumps(["name"])}
+            res = self.app.data.find('items_with_description', req, None)
+            self.assertEqual(1, res.count())
+            self.assertTrue('description' not in res.docs[0])
+            self.assertTrue('name' in res.docs[0])
+
     def test_should_aggregate(self):
         with self.app.app_context():
             self.app.config['ELASTICSEARCH_AUTO_AGGREGATIONS'] = False
@@ -317,6 +331,23 @@ def test_should_aggregate(self):
             req.args = {'aggregations': '0'}
             self.assertFalse(self.app.data.should_aggregate(req))
 
+    def test_should_project(self):
+        with self.app.app_context():
+            req = ParsedRequest()
+            req.args = {'projections': json.dumps(["priority", "urgency", "word_count", "slugline", "highlights"])}
+            self.assertTrue(self.app.data.should_project(req))
+            req.args = {'projections': json.dumps([])}
+            self.assertFalse(self.app.data.should_project(req))
+            req.args = {}
+            self.assertFalse(self.app.data.should_project(req))
+
+    def test_get_projected_fields(self):
+        with self.app.app_context():
+            req = ParsedRequest()
+            req.args = {'projections': json.dumps(["priority", "urgency", "word_count", "slugline", "highlights"])}
+            fields = self.app.data.get_projected_fields(req)
+            self.assertEqual(fields, "priority,urgency,word_count,slugline,highlights")
+
     def test_should_highlight(self):
         with self.app.app_context():
             req = ParsedRequest()