Merge pull request #1306 from UUDigitalHumanitieslab/feature/save-sea…

…rch-history-from-backend

Feature/save search history from backend

backend/api/save_query.py

@@ -0,0 +1,40 @@
+from datetime import datetime, timedelta
+
+IGNORE_KEYS = ['size', 'scroll', 'from', 'aggs']
+'Keys that should be ignored when comparing if two queries are identical'
+
+def should_save_query(user, es_query):
+    '''
+    Whether a query should be saved in the search history for a user
+    '''
+
+    if not user.profile.enable_search_history:
+        return False
+
+    if has_aggregations(es_query):
+        return False
+
+    if any(same_query(es_query, q.query_json) for q in recent_queries(user)):
+        return False
+
+    return True
+
+def recent_queries(user):
+    one_hour_ago = datetime.today() - timedelta(hours = 1)
+    return user.queries.filter(
+        completed__gte=one_hour_ago
+    )
+
+def has_aggregations(es_query):
+    return 'aggs' in es_query
+
+
+def _filter_relevant_keys(es_query):
+    return {
+        key: value
+        for (key, value) in es_query.items()
+        if key not in IGNORE_KEYS
+    }
+
+def same_query(es_query_1, es_query_2):
+    return _filter_relevant_keys(es_query_1) == _filter_relevant_keys(es_query_2)

backend/api/tests/test_save_query.py

@@ -0,0 +1,45 @@
+from django.utils import timezone
+import pytest
+from copy import deepcopy
+
+from visualization.query import MATCH_ALL, set_query_text
+from addcorpus.models import Corpus
+from api.models import Query
+from api.save_query import recent_queries, same_query
+
+
+@pytest.fixture()
+def saved_query(auth_user, db):
+    corpus = Corpus.objects.get(name='small-mock-corpus')
+    return Query.objects.create(
+        query_json=MATCH_ALL,
+        user=auth_user,
+        corpus=corpus,
+        completed=timezone.now(),
+        transferred=10,
+        total_results=10,
+    )
+
+
+def test_recent_queries(auth_user, saved_query):
+    assert saved_query in recent_queries(auth_user)
+
+    saved_query.started = timezone.datetime(2000, 1, 1, 0, 0, tzinfo=timezone.get_current_timezone())
+    saved_query.completed = timezone.datetime(2000, 1, 1, 0, 0, tzinfo=timezone.get_current_timezone())
+    saved_query.save()
+
+    assert saved_query not in recent_queries(auth_user)
+
+def test_same_query():
+    assert same_query(MATCH_ALL, MATCH_ALL)
+
+    q1 = deepcopy(MATCH_ALL)
+    q1.update({
+        'size': 20,
+        'from': 21,
+    })
+
+    assert same_query(q1, MATCH_ALL)
+
+    q2 = set_query_text(MATCH_ALL, 'test')
+    assert not same_query(q2, MATCH_ALL)

backend/es/tests/test_es_forward.py

@@ -1,6 +1,5 @@
 import pytest
-import json
-
+from visualization.query import MATCH_ALL
 from es.search import hits
 
 FORWARD_CASES = {
@@ -85,3 +84,24 @@ def test_es_forwarding_views(scenario, es_forward_client, client, times_user):
 
     if response.status_code == 200:
         assert len(hits(response.data)) == n_hits
+
+def test_search_history_is_saved(mock_corpus, times_user, es_forward_client, client):
+    assert times_user.queries.count() == 0
+
+    client.force_login(times_user)
+
+    search = lambda: client.post(
+        '/api/es/times/_search',
+        {'es_query': MATCH_ALL},
+        content_type='application/json',
+    )
+
+    response = search()
+
+    assert response.status_code == 200
+    assert times_user.queries.count() == 1
+
+    response2 = search()
+
+    assert response2.status_code == 200
+    assert times_user.queries.count() == 1

backend/es/views.py

@@ -1,13 +1,17 @@
+from django.utils import timezone
 from rest_framework.views import APIView
 from rest_framework.response import Response
 from ianalyzer.elasticsearch import elasticsearch
-from es.search import get_index
+from es.search import get_index, total_hits, hits
 import logging
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.exceptions import APIException
 from addcorpus.permissions import CorpusAccessPermission
 from tag.filter import handle_tags_in_request
 from tag.permissions import CanSearchTags
+from api.save_query import should_save_query
+from addcorpus.models import Corpus
+from api.models import Query
 
 logger = logging.getLogger(__name__)
 
@@ -52,6 +56,8 @@ def post(self, request, *args, **kwargs):
             **get_query_parameters(request)
         }
 
+        history_obj = self._save_query_started(request, corpus_name, query)
+
         try:
             results = client.search(
                 index=index,
@@ -62,4 +68,22 @@ def post(self, request, *args, **kwargs):
             logger.exception(e)
             raise APIException('Search failed')
 
+        if history_obj and results:
+            self._save_query_done(history_obj, results)
+
         return Response(results)
+
+    def _save_query_started(self, request, corpus_name, es_query):
+        if should_save_query(request.user, es_query):
+            corpus = Corpus.objects.get(name=corpus_name)
+            return Query.objects.create(
+                user=request.user,
+                corpus=corpus,
+                query_json=es_query,
+            )
+
+    def _save_query_done(self, query, results):
+        query.completed = timezone.now()
+        query.total_results = total_hits(results)
+        query.transferred = len(hits(results))
+        query.save()

frontend/src/app/services/api.service.ts

@@ -63,11 +63,6 @@ export class ApiService {
         return this.http.post('/api/search_history/delete_all/', {});
     }
 
-    // General / misc
-    public saveQuery(options: QueryDb) {
-        return this.http.post('/api/search_history/', options).toPromise();
-    }
-
     public searchHistory() {
         return this.http.get<QueryDb[]>('/api/search_history/').toPromise();
     }

frontend/src/app/services/query.service.ts

@@ -8,11 +8,6 @@ export class QueryService {
 
     constructor(private apiRetryService: ApiRetryService) { }
 
-    save(query: QueryDb): Promise<any> {
-        return this.apiRetryService.requireLogin(api => api.saveQuery(query));
-    }
-
-
     /**
      * Retrieve saved queries
      */

frontend/src/app/services/search.service.ts

@@ -2,21 +2,17 @@ import { Injectable } from '@angular/core';
 
 import { ApiService } from './api.service';
 import { ElasticSearchService } from './elastic-search.service';
-import { QueryService } from './query.service';
 import {
     Corpus, QueryModel, SearchResults,
-    AggregateQueryFeedback, QueryDb, User
+    AggregateQueryFeedback
 } from '../models/index';
-import { AuthService } from './auth.service';
 
 
 @Injectable()
 export class SearchService {
     constructor(
         private apiService: ApiService,
-        private authService: AuthService,
         private elasticSearchService: ElasticSearchService,
-        private queryService: QueryService,
     ) {
         window['apiService'] = this.apiService;
     }
@@ -40,14 +36,8 @@ export class SearchService {
 
     public async search(queryModel: QueryModel
     ): Promise<SearchResults> {
-        const user = await this.authService.getCurrentUserPromise();
-        const request = () => this.elasticSearchService.search(queryModel);
-
-        const resultsPromise = user.enableSearchHistory ?
-            this.searchAndSave(queryModel, user, request) :
-            request();
-
-        return resultsPromise.then(results =>
+        const request = this.elasticSearchService.search(queryModel);
+        return request.then(results =>
             this.filterResultsFields(results, queryModel)
         );
     }
@@ -78,34 +68,6 @@ export class SearchService {
         );
     }
 
-    /** execute a search request and save the action to the search history log */
-    private searchAndSave(queryModel: QueryModel, user: User, searchRequest: () => Promise<SearchResults>): Promise<SearchResults> {
-        return this.recordTime(searchRequest).then(([results, started, completed]) => {
-            this.saveQuery(queryModel, user, results, started, completed);
-            return results;
-        });
-    }
-
-    /** execute a promise while noting the start and end time */
-    private recordTime<T>(makePromise: () => Promise<T>): Promise<[result: T, started: Date, completed: Date]> {
-        const started = new Date(Date.now());
-
-        return makePromise().then(result => {
-            const completed = new Date(Date.now());
-            return [result, started, completed];
-        });
-    }
-
-    /** save query data to search history */
-    private saveQuery(queryModel: QueryModel, user: User, results: SearchResults, started: Date, completed: Date) {
-        const esQuery = queryModel.toEsQuery();
-        const query = new QueryDb(esQuery, queryModel.corpus.name, user.id);
-        query.started = started;
-        query.total_results = results.total.value;
-        query.completed = completed;
-        this.queryService.save(query);
-    }
-
     /** filter search results for fields included in resultsOverview of the corpus */
     private filterResultsFields(results: SearchResults, queryModel: QueryModel): SearchResults {
         return {