support elastic 7 (#3243)

* sort contacts using keyword fields

* make all query string queries ignore format errors

* use elastic7 for e2e

* use updated server core

* use 1 test server worker

* save

* update test server gunicorn

* dump server log on error

* filter superdesk logs on failure

* use 2 backend workers

* fix package specs

* add sentry dsn to test server settings

* enable sentry on travis

* sync mongo uris with dbnames

* use PR branch for test server

* fix contacts sorting

* Revert "fix contacts sorting"

This reverts commit b66efea.

* use latest core

.travis.yml

@@ -17,6 +17,9 @@ before_script:
         npm run start-test-server
       fi
 
+after_failure:
+    - cd test-server && docker-compose logs superdesk | grep -i error
+
 jobs:
     include:
         - script: npm run test

package.json

@@ -127,7 +127,7 @@
   },
   "devDependencies": {
     "@types/moment": "^2.13.0",
-    "@types/moment-timezone": "^0.5.12",
+    "@types/moment-timezone": "^0.5.13",
     "@types/webpack-env": "^1.13.9",
     "btoa": "^1.1.2",
     "enzyme": "3.9.0",
@@ -144,8 +144,8 @@
     "karma-webpack": "^2.0.13",
     "protractor": "5.4.2",
     "react-addons-test-utils": "^15.6.0",
-    "react-test-renderer": "^16.2.0",
-    "request": "^2.83.0",
+    "react-test-renderer": "^16.13.1",
+    "request": "^2.88.2",
     "superdesk-code-style": "^1.1.1",
     "typescript-eslint-parser": "^18.0.0"
   },

scripts/apps/archive/services/FamilyService.ts

@@ -179,7 +179,7 @@ export function FamilyService(api, desks) {
                 queryString = {
                     query_string: {
                         query: queryRelatedItem.join(' '),
-                        lenient: false,
+                        lenient: true,
                         default_operator: 'OR',
                     },
                 };
@@ -198,7 +198,7 @@ export function FamilyService(api, desks) {
             queryString = {
                 query_string: {
                     query: 'slugline.phrase:("' + sanitizedKeyword + '")',
-                    lenient: false,
+                    lenient: true,
                 },
             };
         }

scripts/apps/archive/tests/family.spec.ts

@@ -158,7 +158,7 @@ describe('family service', () => {
                                 ],
                             },
                             query: {
-                                query_string: {query: 'slugline.phrase:("test")', lenient: false},
+                                query_string: {query: 'slugline.phrase:("test")', lenient: true},
                             },
                         },
                     },

scripts/apps/contacts/directives/ContactsSortBarDirective.ts

@@ -8,7 +8,7 @@ class LinkFunction extends BaseSortBar {
         this.contacts = contacts;
         this.scope.canSort = super.canSort.bind(this);
         this.scope.sortOptions = contacts.sortOptions;
-        sort.setSort('last_name', contacts.sortOptions);
+        sort.setSort(this.scope.sortOptions[0].field, contacts.sortOptions);
         super.getActive();
     }
 }

scripts/apps/contacts/services/ContactsService.ts

@@ -34,8 +34,8 @@ export class ContactsService implements IContactsService {
         this.$location = $location;
         this.sort = sort;
         this.sortOptions = [
-            {field: 'last_name', label: gettext('Person (Last Name)'), defaultDir: 'asc'},
-            {field: 'organisation', label: gettext('Organisation'), defaultDir: 'asc'},
+            {field: 'last_name.keyword', label: gettext('Person (Last Name)'), defaultDir: 'asc'},
+            {field: 'organisation.keyword', label: gettext('Organisation'), defaultDir: 'asc'},
             {field: '_created', label: gettext('Created')},
             {field: '_updated', label: gettext('Updated')},
         ];

scripts/apps/monitoring/services/CardsService.ts

@@ -233,7 +233,7 @@ export function CardsService(search, session, desks, $location) {
         filterQueryByCustomQuery(query, card);
 
         if (queryString) {
-            query.filter({query: {query_string: {query: queryString, lenient: false}}});
+            query.filter({query: {query_string: {query: queryString, lenient: true}}});
             criteria.es_highlight = search.getElasticHighlight();
         }
 

scripts/apps/monitoring/tests/monitoring.spec.ts

@@ -136,7 +136,7 @@ describe('monitoring', () => {
 
             criteria = cards.criteria(card, 'foo');
             expect(criteria.source.query.filtered.filter.and).toContain({
-                query: {query_string: {query: 'foo', lenient: false}},
+                query: {query_string: {query: 'foo', lenient: true}},
             });
         }));
 

scripts/apps/search/services/SearchService.ts

@@ -514,7 +514,7 @@ export function SearchService($location, session, multi,
             if (queryString) {
                 criteria.query.filtered.query = {query_string: {
                     query: queryString,
-                    lenient: false,
+                    lenient: true,
                     default_operator: 'AND',
                 }};
             }

spec/helpers/monitoring.ts

@@ -188,7 +188,7 @@ class Monitoring {
          * item of that type from group
          *
          * @param {Number} group
-         * @param {Number|Object} item
+         * @param {Number|Object|String} item
          * @return {WebElement}
          */
         this.getItem = function(group, item) {
@@ -199,9 +199,16 @@ class Monitoring {
             if (item.type) {
                 return all.filter((elem) =>
                     elem.all(by.className('filetype-icon-' + item.type)).count()).get(item.index || 0);
+            } else if (Number.isInteger(item)) {
+                return all.get(item);
+            } else { // use slugline filter
+                return all.filter((elem) =>
+                    elem.element(s(['field--slugline'])).isPresent(),
+                ).filter((elem) =>
+                    elem.element(s(['field--slugline'])).getText()
+                        .then((text) => text.toLowerCase().includes(item.toLowerCase())),
+                ).first();
             }
-
-            return all.get(item);
         };
 
         this.getGroupItems = function(group) {

spec/package_spec.ts

@@ -12,7 +12,7 @@ describe('package', () => {
     });
 
     it('increment package version', () => {
-        monitoring.actionOnItem('Edit', 3, 0);
+        monitoring.actionOnItem('Edit', 3, 'package3');
         // Add item to current package.
         monitoring.actionOnItemSubmenu('Add to current', 'main', 2, 0);
         // Save package
@@ -24,7 +24,7 @@ describe('package', () => {
     });
 
     it('add to current package removed', () => {
-        monitoring.actionOnItem('Edit', 3, 0);
+        monitoring.actionOnItem('Edit', 3, 'package3');
         monitoring.actionOnItemSubmenu('Add to current', 'main', 2, 0);
         // Open menu.
         var menu = monitoring.openItemMenu(2, 0);
@@ -36,7 +36,7 @@ describe('package', () => {
     });
 
     it('reorder group package items', () => {
-        monitoring.actionOnItem('Edit', 3, 0);
+        monitoring.actionOnItem('Edit', 3, 'package3');
         monitoring.actionOnItemSubmenu('Add to current', 'main', 2, 0);
         monitoring.actionOnItemSubmenu('Add to current', 'story', 3, 2);
         // Reorder item on package
@@ -55,14 +55,14 @@ describe('package', () => {
     it('create package by combining an item with open item', () => {
         monitoring.openAction(2, 1);
         browser.sleep(500);
-        monitoring.actionOnItem('Combine with current', 3, 0);
+        monitoring.actionOnItem('Combine with current', 3, 'package3');
         expect(authoring.getGroupItems('MAIN').count()).toBe(2);
     });
 
     it('add multiple items to package', () => {
-        monitoring.actionOnItem('Edit', 3, 0);
+        monitoring.actionOnItem('Edit', 3, 'package3');
         monitoring.selectItem(2, 0);
-        monitoring.selectItem(3, 1);
+        monitoring.selectItem(3, 'package2');
         browser.sleep(200);
         multiAction('Add to Current Package');
         browser.sleep(200);
@@ -88,7 +88,7 @@ describe('package', () => {
     xit('sets package item label', () => {
         workspace.selectDesk('Politic Desk');
         expect(monitoring.getTextItem(3, 1)).toBe('package2');
-        monitoring.actionOnItem('Edit', 3, 1);
+        monitoring.actionOnItem('Edit', 3, 'package2');
         monitoring.getPackageItemActionDropdown(0).click();
         browser.actions()
             .mouseMove(monitoring.getPackageItemLabelEntry())

test-server/docker-compose.yml

@@ -13,7 +13,7 @@ services:
             - /data/db
 
     elastic:
-        image: elasticsearch:2.4-alpine
+        image: elasticsearch:7.6.2
         ports:
             - "9200:9200"
         environment:
@@ -41,3 +41,4 @@ services:
             - WEB_WORKERS=2
             - WEB_TIMEOUT=10
             - WEB_LOG_LEVEL=info
+            - SENTRY_DSN

test-server/gunicorn_config.py

@@ -1,5 +1,7 @@
 import os
 
+reload = bool(os.environ.get('WEB_RELOAD'))
+
 bind = '0.0.0.0:%s' % os.environ.get('PORT', '5000')
 
 workers = int(os.environ.get('WEB_WORKERS', 2))
@@ -10,4 +12,3 @@
 if bool(os.environ.get('SUPERDESK_DEBUG')):
     accesslog = '-'
     access_log_format = '%(m)s %(U)s status=%(s)s time=%(T)ss size=%(B)sb'
-

test-server/requirements.txt

@@ -1,4 +1,3 @@
-gunicorn==19.7.1
+gunicorn==20.0.4
 honcho==1.0.1
-
 git+git://github.com/superdesk/superdesk-core.git@develop#egg=Superdesk-Core

tslint.json

@@ -57,6 +57,7 @@
     "rulesDirectory": [],
     "linterOptions": {
         "exclude": [
+            "test-server/**",
             "node_modules/**",
             "scripts/extensions/**/node_modules/**",
             "end-to-end-testing-helpers/dist/**",