From 9adf15e616e5697e7ad588563473e4e32b99a75a Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 14 May 2024 17:42:49 +0200 Subject: [PATCH 01/14] Allow to choose entity type to index for search + better logging --- src/common/helpers/search.ts | 493 ++++++++++++++++++++--------------- 1 file changed, 289 insertions(+), 204 deletions(-) diff --git a/src/common/helpers/search.ts b/src/common/helpers/search.ts index e1092311e..a02d9a664 100644 --- a/src/common/helpers/search.ts +++ b/src/common/helpers/search.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-undefined */ /* * Copyright (C) 2016 Sean Burke * @@ -52,6 +53,79 @@ function sanitizeEntityType(type) { type IndexableEntities = EntityTypeString | 'Editor' | 'Collection' | 'Area'; const commonProperties = ['bbid', 'id', 'name', 'type', 'disambiguation']; +const indexMappings = { + mappings: { + _default_: { + properties: { + aliases: { + properties: { + name: { + fields: { + autocomplete: { + analyzer: 'edge', + type: 'text' + }, + search: { + analyzer: 'trigrams', + type: 'text' + } + }, + type: 'text' + } + } + }, + authors: { + analyzer: 'trigrams', + type: 'text' + }, + disambiguation: { + analyzer: 'trigrams', + type: 'text' + } + } + } + }, + settings: { + analysis: { + analyzer: { + edge: { + filter: [ + 'asciifolding', + 'lowercase' + ], + tokenizer: 'edge_ngram_tokenizer', + type: 'custom' + }, + trigrams: { + filter: [ + 'asciifolding', + 'lowercase' + ], + tokenizer: 'trigrams', + type: 'custom' + } + }, + tokenizer: { + edge_ngram_tokenizer: { + max_gram: 10, + min_gram: 2, + token_chars: [ + 'letter', + 'digit' + ], + type: 'edge_ngram' + }, + trigrams: { + max_gram: 3, + min_gram: 1, + type: 'ngram' + } + } + }, + 'index.mapping.ignore_malformed': true + } +}; + /* We don't currently want to index the entire Model in ElasticSearch, which contains a lot of fields we don't use as well as some internal props (_pivot props) This utility function prepares the Model into a minimal object that will be indexed @@ -65,16 +139,25 @@ export function getDocumentToIndex(entity:any, entityType: IndexableEntities) { default: break; } - let aliases = entity.related('aliasSet')?.related('aliases')?.toJSON({ignorePivot: true, visible: 'name'}); + let aliases = entity + .related('aliasSet') + ?.related('aliases') + ?.toJSON({ignorePivot: true, visible: 'name'}); if (!aliases) { // Some models don't have the same aliasSet structure, i.e. Collection, Editor, Area, … const name = entity.get('name'); aliases = {name}; } - const identifiers = entity.related('identifierSet')?.related('identifiers')?.toJSON({ignorePivot: true, visible: 'value'}); + const identifiers = entity + .related('identifierSet') + ?.related('identifiers') + ?.toJSON({ignorePivot: true, visible: 'value'}); return { - ...entity.toJSON({ignorePivot: true, visible: commonProperties.concat(additionalProperties)}), + ...entity.toJSON({ + ignorePivot: true, + visible: commonProperties.concat(additionalProperties) + }), aliases, identifiers: identifiers ?? null }; @@ -190,9 +273,13 @@ export async function _bulkIndexEntities(entities) { operationSucceeded = true; // eslint-disable-next-line no-await-in-loop - const response = await _client.bulk({ - body: bulkOperations - }).catch(error => { log.error('error bulk indexing entities for search:', error); }); + const response = await _client + .bulk({ + body: bulkOperations + }) + .catch((error) => { + log.error('error bulk indexing entities for search:', error); + }); /* * In case of failed index operations, the promise won't be rejected; @@ -283,114 +370,59 @@ export function indexEntity(entity) { const entityType = entity.get('type'); const document = getDocumentToIndex(entity, entityType); if (entity) { - return _client.index({ - body: document, - id: entity.get('bbid') || entity.get('id'), - index: _index, - type: snakeCase(entityType) - }).catch(error => { log.error('error indexing entity for search:', error); }); + return _client + .index({ + body: document, + id: entity.get('bbid') || entity.get('id'), + index: _index, + type: snakeCase(entityType) + }) + .catch((error) => { + log.error('error indexing entity for search:', error); + }); } } export function deleteEntity(entity) { - return _client.delete({ - id: entity.bbid ?? entity.id, - index: _index, - type: snakeCase(entity.type) - }).catch(error => { log.error('error deleting entity from index:', error); }); + return _client + .delete({ + id: entity.bbid ?? entity.id, + index: _index, + type: snakeCase(entity.type) + }) + .catch((error) => { + log.error('error deleting entity from index:', error); + }); } export function refreshIndex() { - return _client.indices.refresh({index: _index}).catch(error => { log.error('error refreshing search index:', error); }); + return _client.indices.refresh({index: _index}).catch((error) => { + log.error('Error refreshing search index:', error); + }); } -export async function generateIndex(orm) { +export async function generateIndex(orm, entityType: IndexableEntities | 'allEntities' = 'allEntities', recreateIndex = false) { const {Area, Author, Edition, EditionGroup, Editor, Publisher, Series, UserCollection, Work} = orm; - const indexMappings = { - mappings: { - _default_: { - properties: { - aliases: { - properties: { - name: { - fields: { - autocomplete: { - analyzer: 'edge', - type: 'text' - }, - search: { - analyzer: 'trigrams', - type: 'text' - } - }, - type: 'text' - } - } - }, - authors: { - analyzer: 'trigrams', - type: 'text' - }, - disambiguation: { - analyzer: 'trigrams', - type: 'text' - } - } - } - }, - settings: { - analysis: { - analyzer: { - edge: { - filter: [ - 'asciifolding', - 'lowercase' - ], - tokenizer: 'edge_ngram_tokenizer', - type: 'custom' - }, - trigrams: { - filter: [ - 'asciifolding', - 'lowercase' - ], - tokenizer: 'trigrams', - type: 'custom' - } - }, - tokenizer: { - edge_ngram_tokenizer: { - max_gram: 10, - min_gram: 2, - token_chars: [ - 'letter', - 'digit' - ], - type: 'edge_ngram' - }, - trigrams: { - max_gram: 3, - min_gram: 1, - type: 'ngram' - } - } - }, - 'index.mapping.ignore_malformed': true - } - }; - // First, drop index and recreate - const mainIndexExistsRequest = await _client.indices.exists({index: _index}); - const mainIndexExists = mainIndexExistsRequest?.body; + const allEntities = entityType === 'allEntities'; + const mainIndexExists = await _client.indices.exists({ + index: _index + })?.body; - if (mainIndexExists) { + const shouldRecreateIndex = mainIndexExists && (recreateIndex || allEntities); + + if (shouldRecreateIndex) { + // Drop index and recreate await _client.indices.delete({index: _index}); } + if (!mainIndexExists || shouldRecreateIndex) { + // Create the index if missing or re-create it if we just deleted it + await _client.indices.create({body: indexMappings, index: _index}); + } - await _client.indices.create( - {body: indexMappings, index: _index} - ); + log.notice(`Starting indexing of ${entityType}`); + const entityBehaviors = []; const baseRelations = [ 'annotation', 'defaultAlias', @@ -398,32 +430,54 @@ export async function generateIndex(orm) { 'identifierSet.identifiers' ]; - const entityBehaviors = [ - { + if (allEntities || entityType === 'Author' || entityType === 'Work') { + entityBehaviors.push({ model: Author, - relations: [ - 'gender', - 'beginArea', - 'endArea' - ] - }, - { + relations: ['gender', 'beginArea', 'endArea'], + type: 'Author' + }); + } + if (allEntities || entityType === 'Edition') { + entityBehaviors.push({ model: Edition, - relations: [ - 'editionGroup', - 'editionFormat', - 'editionStatus' - ] - }, - {model: EditionGroup, relations: []}, - {model: Publisher, relations: ['area']}, - {model: Series, relations: ['seriesOrderingType']}, - {model: Work, relations: ['relationshipSet.relationships.type']} - ]; + relations: ['editionGroup', 'editionFormat', 'editionStatus'], + type: 'Edition' + }); + } + if (allEntities || entityType === 'EditionGroup') { + entityBehaviors.push({ + model: EditionGroup, + relations: [], + type: 'EditionGroup' + }); + } + if (allEntities || entityType === 'Publisher') { + entityBehaviors.push({ + model: Publisher, + relations: ['area'], + type: 'Publisher' + }); + } + if (allEntities || entityType === 'Series') { + entityBehaviors.push({ + model: Series, + relations: ['seriesOrderingType'], + type: 'Series' + }); + } + if (allEntities || entityType === 'Work') { + log.info('Also indexing Author entities'); + entityBehaviors.push({ + model: Work, + relations: ['relationshipSet.relationships.type'], + type: 'Work' + }); + } // Update the indexed entries for each entity type - const behaviorPromise = entityBehaviors.map( - (behavior) => behavior.model.forge() + const behaviorPromise = entityBehaviors.map(async (behavior) => ({ + results: await behavior.model + .forge() .query((qb) => { qb.where('master', true); qb.whereNotNull('data_id'); @@ -431,96 +485,117 @@ export async function generateIndex(orm) { .fetchAll({ withRelated: baseRelations.concat(behavior.relations) }) - ); + .catch(log.error), + type: behavior.type + })); + log.info(`Finished fetching entities from database for type ${entityType}`); + const entityLists = await Promise.all(behaviorPromise); - /* eslint-disable @typescript-eslint/no-unused-vars */ - const entityFetchOrder:EntityTypeString[] = ['Author', 'Edition', 'EditionGroup', 'Publisher', 'Series', 'Work']; - const [authorsCollection, - editionCollection, - editionGroupCollection, - publisherCollection, - seriesCollection, - workCollection] = entityLists; - /* eslint-enable @typescript-eslint/no-unused-vars */ - const listIndexes = []; - workCollection.forEach(workEntity => { - const relationshipSet = workEntity.related('relationshipSet'); - if (relationshipSet) { - const authorWroteWorkRels = relationshipSet.related('relationships')?.filter(relationshipModel => relationshipModel.get('typeId') === 8); - const authorNames = []; - authorWroteWorkRels.forEach(relationshipModel => { - // Search for the Author in the already fetched BookshelfJS Collection - const source = authorsCollection.get(relationshipModel.get('sourceBbid')); - const name = source?.related('defaultAlias')?.get('name'); - if (name) { - authorNames.push(name); - } - }); - workEntity.set('authors', authorNames); - } - }); + if (allEntities || entityType === 'Work') { + log.info('Attaching author names to Work entities'); + const authorCollection = entityLists.find( + ({type}) => type === 'Author' + ); + const workCollection = entityLists.find(({type}) => type === 'Work'); + workCollection?.results.forEach((workEntity) => { + const relationshipSet = workEntity.related('relationshipSet'); + if (relationshipSet) { + const authorWroteWorkRels = relationshipSet + .related('relationships') + ?.filter( + (relationshipModel) => + relationshipModel.get('typeId') === 8 + ); + const authorNames = []; + authorWroteWorkRels.forEach((relationshipModel) => { + // Search for the Author in the already fetched BookshelfJS Collection + const source = authorCollection.get( + relationshipModel.get('sourceBbid') + ); + const name = source?.related('defaultAlias')?.get('name'); + if (name) { + authorNames.push(name); + } + }); + workEntity.set('authors', authorNames); + } + }); + } + + const listIndexes = []; // Index all the entities - entityLists.forEach((entityList, idx) => { - const entityType:EntityTypeString = entityFetchOrder[idx]; - const listArray = entityList.map(entity => getDocumentToIndex(entity, entityType)); + entityLists.forEach((entityList) => { + const listArray = entityList.results.map((entity) => + getDocumentToIndex(entity, entityList.type)); listIndexes.push(_processEntityListForBulk(listArray)); }); - await Promise.all(listIndexes); + if (listIndexes.length) { + log.info(`Indexing documents for entity type ${entityType}`); + await Promise.all(listIndexes); + log.info(`Finished indexing entity documents for entity type ${entityType}`); + } + + if (allEntities || entityType === 'Area') { + log.info('Indexing Areas'); - const areaCollection = await Area.forge() - .fetchAll(); + const areaCollection = await Area.forge().fetchAll(); - const areas = areaCollection.toJSON({omitPivot: true}); + const areas = areaCollection.toJSON({omitPivot: true}); - /** To index names, we use aliases.name and type, which Areas don't have. - * We massage the area to return a similar format as BB entities - */ - const processedAreas = areas.map((area) => ({ - aliases: [ - {name: area.name} - ], - id: area.gid, - type: 'Area' - })); - await _processEntityListForBulk(processedAreas); - - const editorCollection = await Editor.forge() - // no bots - .where('type_id', 1) - .fetchAll(); - const editors = editorCollection.toJSON({omitPivot: true}); - - /** To index names, we use aliases.name and type, which Editors don't have. - * We massage the editor to return a similar format as BB entities - */ - const processedEditors = editors.map((editor) => ({ - aliases: [ - {name: editor.name} - ], - id: editor.id, - type: 'Editor' - })); - await _processEntityListForBulk(processedEditors); - - const userCollections = await UserCollection.forge().where({public: true}) - .fetchAll(); - const userCollectionsJSON = userCollections.toJSON({omitPivot: true}); - - /** To index names, we use aliases.name and type, which UserCollections don't have. - * We massage the editor to return a similar format as BB entities - */ - const processedCollections = userCollectionsJSON.map((collection) => ({ - aliases: [ - {name: collection.name} - ], - id: collection.id, - type: 'Collection' - })); - await _processEntityListForBulk(processedCollections); + /** To index names, we use aliases.name and type, which Areas don't have. + * We massage the area to return a similar format as BB entities + */ + const processedAreas = areas.map((area) => ({ + aliases: [{name: area.name}], + id: area.gid, + type: 'Area' + })); + await _processEntityListForBulk(processedAreas); + log.info('Finished indexing Areas'); + } + if (allEntities || entityType === 'Editor') { + log.info('Indexing Editors'); + const editorCollection = await Editor.forge() + // no bots + .where('type_id', 1) + .fetchAll(); + const editors = editorCollection.toJSON({omitPivot: true}); + + /** To index names, we use aliases.name and type, which Editors don't have. + * We massage the editor to return a similar format as BB entities + */ + const processedEditors = editors.map((editor) => ({ + aliases: [{name: editor.name}], + id: editor.id, + type: 'Editor' + })); + await _processEntityListForBulk(processedEditors); + log.info('Finished indexing Editors'); + } + + if (allEntities || entityType === 'Collection') { + log.info('Indexing Collections'); + const userCollections = await UserCollection.forge() + .where({public: true}) + .fetchAll(); + const userCollectionsJSON = userCollections.toJSON({omitPivot: true}); + /** To index names, we use aliases.name and type, which UserCollections don't have. + * We massage the editor to return a similar format as BB entities + */ + const processedCollections = userCollectionsJSON.map((collection) => ({ + aliases: [{name: collection.name}], + id: collection.id, + type: 'Collection' + })); + await _processEntityListForBulk(processedCollections); + log.info('Finished indexing Collections'); + } + log.info('Refreshing search index'); await refreshIndex(); + log.notice('Search indexing finished succesfully'); } export async function checkIfExists(orm, name, type) { @@ -529,7 +604,9 @@ export async function checkIfExists(orm, name, type) { bookshelf.transaction(async (transacting) => { try { const result = await orm.func.alias.getBBIDsWithMatchingAlias( - transacting, snakeCase(type), name + transacting, + snakeCase(type), + name ); resolve(result); } @@ -549,9 +626,13 @@ export async function checkIfExists(orm, name, type) { 'revision.revision' ]; const processedResults = await Promise.all( - bbids.map( - bbid => orm.func.entity.getEntity(orm, upperFirst(camelCase(type)), bbid, baseRelations) - ) + bbids.map((bbid) => + orm.func.entity.getEntity( + orm, + upperFirst(camelCase(type)), + bbid, + baseRelations + )) ); return processedResults; @@ -580,7 +661,11 @@ export function searchByName(orm, name, type, size, from) { index: _index, type: sanitizedEntityType }; - if (sanitizedEntityType === 'work' || (Array.isArray(sanitizedEntityType) && sanitizedEntityType.includes('work'))) { + if ( + sanitizedEntityType === 'work' || + (Array.isArray(sanitizedEntityType) && + sanitizedEntityType.includes('work')) + ) { dslQuery.body.query.multi_match.fields.push('authors'); } From e27fe3b423014c10446e8dcd537c94225698208a Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 14 May 2024 17:45:26 +0200 Subject: [PATCH 02/14] Update search routes to allow choosing entity + change file to typescript --- src/server/routes/{search.js => search.tsx} | 31 +++++++++++++-------- 1 file changed, 20 insertions(+), 11 deletions(-) rename src/server/routes/{search.js => search.tsx} (80%) diff --git a/src/server/routes/search.js b/src/server/routes/search.tsx similarity index 80% rename from src/server/routes/search.js rename to src/server/routes/search.tsx index 4aaa03694..804af63b6 100644 --- a/src/server/routes/search.js +++ b/src/server/routes/search.tsx @@ -24,7 +24,7 @@ import * as handler from '../helpers/handler'; import * as propHelpers from '../../client/helpers/props'; import * as search from '../../common/helpers/search'; -import {keys as _keys, snakeCase as _snakeCase, isNil} from 'lodash'; +import {keys as _keys, snakeCase as _snakeCase, camelCase, isNil, isString, upperFirst} from 'lodash'; import {escapeProps, generateProps} from '../helpers/props'; import Layout from '../../client/containers/layout'; @@ -34,6 +34,7 @@ import ReactDOMServer from 'react-dom/server'; import SearchPage from '../../client/components/pages/search'; import express from 'express'; import target from '../templates/target'; +import { EntityTypeString } from 'bookbrainz-data/lib/types/entity'; const router = express.Router(); @@ -46,9 +47,9 @@ const {REINDEX_SEARCH_SERVER} = PrivilegeType; router.get('/', async (req, res, next) => { const {orm} = req.app.locals; const query = req.query.q ?? ''; - const type = req.query.type || 'allEntities'; - const size = req.query.size ? parseInt(req.query.size, 10) : 20; - const from = req.query.from ? parseInt(req.query.from, 10) : 0; + const type = String(req.query.type) || 'allEntities'; + const size = req.query.size ? parseInt(String(req.query.size), 10) : 20; + const from = req.query.from ? parseInt(String(req.query.from), 10) : 0; try { let searchResults = { initialResults: [], @@ -59,9 +60,11 @@ router.get('/', async (req, res, next) => { // get 1 more results to check nextEnabled const searchResponse = await search.searchByName(orm, query, _snakeCase(type), size + 1, from); const {results: entities} = searchResponse; + const initialResults = entities.filter(entity => !isNil(entity)); searchResults = { - initialResults: entities.filter(entity => !isNil(entity)), - query + initialResults, + query, + total: initialResults.length }; } @@ -102,7 +105,7 @@ router.get('/', async (req, res, next) => { router.get('/search', (req, res) => { const {orm} = req.app.locals; const query = req.query.q; - const type = req.query.type || 'allEntities'; + const type = req.query.type?.toString() || 'allEntities'; const {size, from} = req.query; @@ -119,7 +122,7 @@ router.get('/autocomplete', (req, res) => { const {orm} = req.app.locals; const query = req.query.q; const type = req.query.type || 'allEntities'; - const size = req.query.size || 42; + const size = Number(req.query.size) || 42; const searchPromise = search.autocomplete(orm, query, type, size); @@ -134,7 +137,7 @@ router.get('/exists', (req, res) => { const {orm} = req.app.locals; const {q, type} = req.query; - const searchPromise = search.checkIfExists(orm, q, _snakeCase(type)); + const searchPromise = search.checkIfExists(orm, q, _snakeCase(String(type))); handler.sendPromiseResult(res, searchPromise); }); @@ -145,10 +148,16 @@ router.get('/exists', (req, res) => { * @throws {error.PermissionDeniedError} - Thrown if user is not admin. */ router.get('/reindex', auth.isAuthenticated, auth.isAuthorized(REINDEX_SEARCH_SERVER), async (req, res) => { + req.socket.setTimeout(600000); const {orm} = req.app.locals; - await search.generateIndex(orm); + const type = isString(req.query.type) ? upperFirst(camelCase(req.query.type)) : "allEntities"; + try { + await search.generateIndex(orm, type as EntityTypeString ); + return handler.sendPromiseResult(res, {success: true}); + } catch (err) { + return error.sendErrorAsJSON(res, new error.SiteError("Cannot index entites for search, something went wrong", req)); + } - handler.sendPromiseResult(res, {success: true}); }); router.get('/entity/:bbid', async (req, res) => { From 96080e58f7cb385169f62d128a641b864c365e17 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 14 May 2024 17:56:25 +0200 Subject: [PATCH 03/14] Add search admin page --- src/client/components/pages/searchAdmin.tsx | 132 +++++++++++++++++++ src/client/controllers/admin/searchAdmin.tsx | 51 +++++++ webpack.client.js | 1 + 3 files changed, 184 insertions(+) create mode 100644 src/client/components/pages/searchAdmin.tsx create mode 100644 src/client/controllers/admin/searchAdmin.tsx diff --git a/src/client/components/pages/searchAdmin.tsx b/src/client/components/pages/searchAdmin.tsx new file mode 100644 index 000000000..badb6b8c1 --- /dev/null +++ b/src/client/components/pages/searchAdmin.tsx @@ -0,0 +1,132 @@ +/* eslint-disable react/jsx-no-bind */ + +import {Alert, Button, Card, Spinner} from 'react-bootstrap'; +import React, {useCallback, useState} from 'react'; +import {faCheck, faListCheck, faTriangleExclamation} from '@fortawesome/free-solid-svg-icons'; +import {ENTITY_TYPE_ICONS} from '../../helpers/entity'; +import {FontAwesomeIcon} from '@fortawesome/react-fontawesome'; + + +export default function SearchAdminPage() { + const [loading, setLoading] = useState(false); + const [errorMessage, setErrorMessage] = useState(); + const [success, setSuccess] = useState(false); + const indexEntity = useCallback(async (entityType?:string) => { + let url = '/search/reindex'; + if (entityType) { + url += `?type=${entityType}`; + } + setSuccess(false); + setLoading(true); + try { + const res = await fetch(url, {headers: {'Request-Timeout': '600'}}); + if (!res.ok) { + const body = await res.json(); + const err = body?.error; + throw new Error(err || res.statusText); + } + setSuccess(true); + } + catch (error) { + setErrorMessage(error); + } + setLoading(false); + }, []); + return ( + + + Search indexing + + +
+ + + + + + + + + + +
+
+
+ {loading && <> In progress...} + {success && + { setSuccess(false); }}> + Success + + } + {errorMessage && + { setErrorMessage(''); }}> + {errorMessage} + + } +
+ + + ); +} diff --git a/src/client/controllers/admin/searchAdmin.tsx b/src/client/controllers/admin/searchAdmin.tsx new file mode 100644 index 000000000..b8ee0bce1 --- /dev/null +++ b/src/client/controllers/admin/searchAdmin.tsx @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2023 Shivam Awasthi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +import {extractChildProps, extractLayoutProps} from '../../helpers/props'; +import {AppContainer} from 'react-hot-loader'; +import Layout from '../../containers/layout'; +import React from 'react'; +import ReactDOM from 'react-dom'; +import SearchAdminPage from '../../components/pages/searchAdmin'; + + +const propsTarget = document.getElementById('props'); +const props = propsTarget ? JSON.parse(propsTarget.innerHTML) : {}; + +const markup = ( + + + + + +); + +ReactDOM.hydrate(markup, document.getElementById('target')); + +/* + * As we are not exporting a component, + * we cannot use the react-hot-loader module wrapper, + * but instead directly use webpack Hot Module Replacement API + */ + +if (module.hot) { + module.hot.accept(); +} diff --git a/webpack.client.js b/webpack.client.js index 97e2b6361..7566fee11 100644 --- a/webpack.client.js +++ b/webpack.client.js @@ -21,6 +21,7 @@ const clientConfig = { entry: { adminLogs: ['./controllers/adminLogs'], adminPanel: ['./controllers/admin-panel'], + searchAdmin: ['./controllers/admin/searchAdmin'], collection: ['./controllers/collection/collection'], 'collection/create': ['./controllers/collection/userCollectionForm'], collections: ['./controllers/collections'], From e6b2a2176c2e14e63cc32516cd6a062aed4a8554 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Wed, 15 May 2024 13:27:46 +0200 Subject: [PATCH 04/14] Fix search indexing --- src/common/helpers/search.ts | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/common/helpers/search.ts b/src/common/helpers/search.ts index a02d9a664..f5948e304 100644 --- a/src/common/helpers/search.ts +++ b/src/common/helpers/search.ts @@ -50,7 +50,7 @@ function sanitizeEntityType(type) { return snakeCase(type); } -type IndexableEntities = EntityTypeString | 'Editor' | 'Collection' | 'Area'; +export type IndexableEntities = EntityTypeString | 'Editor' | 'Collection' | 'Area'; const commonProperties = ['bbid', 'id', 'name', 'type', 'disambiguation']; const indexMappings = { @@ -130,8 +130,9 @@ const indexMappings = { which contains a lot of fields we don't use as well as some internal props (_pivot props) This utility function prepares the Model into a minimal object that will be indexed */ -export function getDocumentToIndex(entity:any, entityType: IndexableEntities) { +export function getDocumentToIndex(entity:any) { const additionalProperties = []; + const entityType:IndexableEntities = entity.get('type'); switch (entityType) { case 'Work': additionalProperties.push('authors'); @@ -368,7 +369,7 @@ export async function autocomplete(orm, query, type, size = 42) { // eslint-disable-next-line consistent-return export function indexEntity(entity) { const entityType = entity.get('type'); - const document = getDocumentToIndex(entity, entityType); + const document = getDocumentToIndex(entity); if (entity) { return _client .index({ @@ -409,20 +410,20 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt index: _index })?.body; - const shouldRecreateIndex = mainIndexExists && (recreateIndex || allEntities); + const shouldRecreateIndex = !mainIndexExists || recreateIndex || allEntities; if (shouldRecreateIndex) { - // Drop index and recreate + log.notice('Deleting search index'); await _client.indices.delete({index: _index}); - } - if (!mainIndexExists || shouldRecreateIndex) { - // Create the index if missing or re-create it if we just deleted it + log.notice('Creating new search index'); await _client.indices.create({body: indexMappings, index: _index}); } log.notice(`Starting indexing of ${entityType}`); - const entityBehaviors = []; + const entityBehaviors:Array<{model:any, + relations: string[], + type:string}> = []; const baseRelations = [ 'annotation', 'defaultAlias', @@ -473,10 +474,10 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt type: 'Work' }); } - // Update the indexed entries for each entity type - const behaviorPromise = entityBehaviors.map(async (behavior) => ({ - results: await behavior.model + const entityLists = await Promise.all(entityBehaviors.map((behavior) => { + log.info(`Fetching ${behavior.type} models from the database`); + return behavior.model .forge() .query((qb) => { qb.where('master', true); @@ -485,20 +486,20 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt .fetchAll({ withRelated: baseRelations.concat(behavior.relations) }) - .catch(log.error), - type: behavior.type + // .catch((error) => { + // log.error(error); + // throw error; + // }); })); - log.info(`Finished fetching entities from database for type ${entityType}`); - - const entityLists = await Promise.all(behaviorPromise); + log.info(`Finished fetching entities from database for types ${entityBehaviors.map(({type}) => type).join(', ')}`); if (allEntities || entityType === 'Work') { log.info('Attaching author names to Work entities'); const authorCollection = entityLists.find( - ({type}) => type === 'Author' + (result) => result.model instanceof Author ); - const workCollection = entityLists.find(({type}) => type === 'Work'); - workCollection?.results.forEach((workEntity) => { + const workCollection = entityLists.find((result) => result.model instanceof Work); + workCollection?.forEach((workEntity) => { const relationshipSet = workEntity.related('relationshipSet'); if (relationshipSet) { const authorWroteWorkRels = relationshipSet @@ -526,8 +527,7 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt const listIndexes = []; // Index all the entities entityLists.forEach((entityList) => { - const listArray = entityList.results.map((entity) => - getDocumentToIndex(entity, entityList.type)); + const listArray = entityList.map(getDocumentToIndex); listIndexes.push(_processEntityListForBulk(listArray)); }); From d98675d3153b1abcc6aefc516fc1ccb62350308c Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Wed, 15 May 2024 13:28:49 +0200 Subject: [PATCH 05/14] Add search admin page and route --- .../components/pages/admin-panel-search.tsx | 1 + src/client/components/pages/searchAdmin.tsx | 2 +- src/client/containers/layout.js | 4 +- src/server/routes.js | 2 + src/server/routes/search.tsx | 12 ++-- src/server/routes/searchAdmin.tsx | 64 +++++++++++++++++++ webpack.client.js | 2 +- 7 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 src/server/routes/searchAdmin.tsx diff --git a/src/client/components/pages/admin-panel-search.tsx b/src/client/components/pages/admin-panel-search.tsx index 0c2e4171d..04daf930d 100644 --- a/src/client/components/pages/admin-panel-search.tsx +++ b/src/client/components/pages/admin-panel-search.tsx @@ -145,6 +145,7 @@ class AdminPanelSearchPage extends React.Component {
+

User search

- + - Reindex Search Server + Search Admin ); diff --git a/src/server/routes.js b/src/server/routes.js index 6763c8397..840854b00 100644 --- a/src/server/routes.js +++ b/src/server/routes.js @@ -38,6 +38,7 @@ import relationshipTypesRouter from './routes/relationship-types'; import reviewsRouter from './routes/reviews'; import revisionRouter from './routes/revision'; import revisionsRouter from './routes/revisions'; +import searchAdminRouter from './routes/searchAdmin'; import searchRouter from './routes/search'; import seriesRouter from './routes/entity/series'; import statisticsRouter from './routes/statistics'; @@ -59,6 +60,7 @@ function initRootRoutes(app) { app.use('/statistics', statisticsRouter); app.use('/external-service', externalServiceRouter); app.use('/admin-panel', adminPanelRouter); + app.use('/search-admin', searchAdminRouter); app.use('/admin-logs', adminLogsRouter); app.use('/relationship-type', relationshipTypeRouter); app.use('/relationship-types', relationshipTypesRouter); diff --git a/src/server/routes/search.tsx b/src/server/routes/search.tsx index 804af63b6..44353e2c2 100644 --- a/src/server/routes/search.tsx +++ b/src/server/routes/search.tsx @@ -27,6 +27,7 @@ import * as search from '../../common/helpers/search'; import {keys as _keys, snakeCase as _snakeCase, camelCase, isNil, isString, upperFirst} from 'lodash'; import {escapeProps, generateProps} from '../helpers/props'; +import {EntityTypeString} from 'bookbrainz-data/lib/types/entity'; import Layout from '../../client/containers/layout'; import {PrivilegeType} from '../../common/helpers/privileges-utils'; import React from 'react'; @@ -34,7 +35,6 @@ import ReactDOMServer from 'react-dom/server'; import SearchPage from '../../client/components/pages/search'; import express from 'express'; import target from '../templates/target'; -import { EntityTypeString } from 'bookbrainz-data/lib/types/entity'; const router = express.Router(); @@ -150,14 +150,14 @@ router.get('/exists', (req, res) => { router.get('/reindex', auth.isAuthenticated, auth.isAuthorized(REINDEX_SEARCH_SERVER), async (req, res) => { req.socket.setTimeout(600000); const {orm} = req.app.locals; - const type = isString(req.query.type) ? upperFirst(camelCase(req.query.type)) : "allEntities"; + const type = isString(req.query.type) ? upperFirst(camelCase(req.query.type)) : 'allEntities'; try { - await search.generateIndex(orm, type as EntityTypeString ); + await search.generateIndex(orm, type as search.IndexableEntities); return handler.sendPromiseResult(res, {success: true}); - } catch (err) { - return error.sendErrorAsJSON(res, new error.SiteError("Cannot index entites for search, something went wrong", req)); } - + catch (err) { + return error.sendErrorAsJSON(res, new error.SiteError(`Cannot index entites for search, something went wrong: ${err.toString()}`, req)); + } }); router.get('/entity/:bbid', async (req, res) => { diff --git a/src/server/routes/searchAdmin.tsx b/src/server/routes/searchAdmin.tsx new file mode 100644 index 000000000..465976417 --- /dev/null +++ b/src/server/routes/searchAdmin.tsx @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2023 Shivam Awasthi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +import * as auth from '../helpers/auth'; +import * as propHelpers from '../../client/helpers/props'; + +import {escapeProps, generateProps} from '../helpers/props'; +import Layout from '../../client/containers/layout'; +import {PrivilegeType} from '../../common/helpers/privileges-utils'; +import React from 'react'; +import ReactDOMServer from 'react-dom/server'; +import SearchAdminPage from '../../client/components/pages/searchAdmin'; +import express from 'express'; +import target from '../templates/target'; + + +const {REINDEX_SEARCH_SERVER} = PrivilegeType; + + +const router = express.Router(); + +/** + * Generates React markup for the search admin page that is rendered by the user's + * browser. + */ +router.get('/', auth.isAuthenticated, auth.isAuthorized(REINDEX_SEARCH_SERVER), (req, res, next) => { + try { + const props = generateProps(req, res); + const markup = ReactDOMServer.renderToString( + + + + ); + + return res.send(target({ + markup, + props: escapeProps(props), + script: '/js/searchAdmin.js', + title: 'Search Admin' + })); + } + catch (err) { + return next(err); + } +}); + +export default router; diff --git a/webpack.client.js b/webpack.client.js index 7566fee11..b8bb3cbb4 100644 --- a/webpack.client.js +++ b/webpack.client.js @@ -21,7 +21,7 @@ const clientConfig = { entry: { adminLogs: ['./controllers/adminLogs'], adminPanel: ['./controllers/admin-panel'], - searchAdmin: ['./controllers/admin/searchAdmin'], + searchAdmin: ['./controllers/admin/searchAdmin.tsx'], collection: ['./controllers/collection/collection'], 'collection/create': ['./controllers/collection/userCollectionForm'], collections: ['./controllers/collections'], From d3412412f6ab47d16d72b1cab06160a055a09ff4 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 4 Jun 2024 17:40:14 +0200 Subject: [PATCH 06/14] Chunk entity fetching Otherwise postgres is unhappy with > 65535 query parameters --- src/common/helpers/search.ts | 42 ++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/common/helpers/search.ts b/src/common/helpers/search.ts index f425da549..b0b01b97b 100644 --- a/src/common/helpers/search.ts +++ b/src/common/helpers/search.ts @@ -479,21 +479,35 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt }); } // Update the indexed entries for each entity type - const entityLists = await Promise.all(entityBehaviors.map((behavior) => { + const entityLists = await Promise.all(entityBehaviors.map(async (behavior) => { log.info(`Fetching ${behavior.type} models from the database`); - return behavior.model - .forge() - .query((qb) => { - qb.where('master', true); - qb.whereNotNull('data_id'); - }) - .fetchAll({ - withRelated: baseRelations.concat(behavior.relations) - }) - // .catch((error) => { - // log.error(error); - // throw error; - // }); + const totalCount:number = await behavior.model.query((qb) => { + qb.where('master', true); + qb.whereNotNull('data_id'); + }).count(); + log.info(`${totalCount} ${behavior.type} models in total`); + const maxChunk = 50000; + const collections = []; + for (let i = 0; i < totalCount; i += maxChunk) { + log.info(`Fetching ${behavior.type} models with offset ${i}`); + // eslint-disable-next-line no-await-in-loop + const collection = await behavior.model + .forge() + .query((qb) => { + qb.where('master', true); + qb.whereNotNull('data_id'); + qb.limit(maxChunk); + qb.offset(i); + }) + .fetchAll({ + withRelated: baseRelations.concat(behavior.relations) + }).catch(err => { log.error(err); throw err; }); + collections.push(collection); + } + const [firstCollection, ...otherCollections] = collections; + const otherModels = otherCollections.map(col => col.models).flat(); + firstCollection.push(otherModels); + return firstCollection; })); log.info(`Finished fetching entities from database for types ${entityBehaviors.map(({type}) => type).join(', ')}`); From dc7fb18933dbf7a9184ec7e3aeb6aa76f9f428af Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 4 Jun 2024 18:10:52 +0200 Subject: [PATCH 07/14] Fix search admin buttons Click event listener on the button, not the icon ! --- src/client/components/pages/searchAdmin.tsx | 27 +++++++-------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/client/components/pages/searchAdmin.tsx b/src/client/components/pages/searchAdmin.tsx index 04348e727..c1103ee27 100644 --- a/src/client/components/pages/searchAdmin.tsx +++ b/src/client/components/pages/searchAdmin.tsx @@ -48,67 +48,58 @@ export default function SearchAdminPage() { } } > - - - - - - - - -
From 62427a2eaaa93d0fbfd0317003c484daa070a838 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 4 Jun 2024 18:25:32 +0200 Subject: [PATCH 08/14] Search: only recreate index when requested --- src/common/helpers/search.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/helpers/search.ts b/src/common/helpers/search.ts index b0b01b97b..950bc78eb 100644 --- a/src/common/helpers/search.ts +++ b/src/common/helpers/search.ts @@ -412,9 +412,9 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt const allEntities = entityType === 'allEntities'; const mainIndexExists = await _client.indices.exists({ index: _index - })?.body; + }); - const shouldRecreateIndex = !mainIndexExists || recreateIndex || allEntities; + const shouldRecreateIndex = !mainIndexExists.body || recreateIndex || allEntities; if (shouldRecreateIndex) { log.notice('Deleting search index'); From 11e5a45fbb520657ce74c699268d2968140fbfb3 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 4 Jun 2024 19:04:05 +0200 Subject: [PATCH 09/14] Fix work author indexing --- src/common/helpers/search.ts | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/common/helpers/search.ts b/src/common/helpers/search.ts index 950bc78eb..912b1ea12 100644 --- a/src/common/helpers/search.ts +++ b/src/common/helpers/search.ts @@ -488,6 +488,7 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt log.info(`${totalCount} ${behavior.type} models in total`); const maxChunk = 50000; const collections = []; + // Fetch by chunks of 50.000 entities for (let i = 0; i < totalCount; i += maxChunk) { log.info(`Fetching ${behavior.type} models with offset ${i}`); // eslint-disable-next-line no-await-in-loop @@ -504,19 +505,18 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt }).catch(err => { log.error(err); throw err; }); collections.push(collection); } - const [firstCollection, ...otherCollections] = collections; - const otherModels = otherCollections.map(col => col.models).flat(); - firstCollection.push(otherModels); - return firstCollection; + // Put all models back into a single collection + const allModels = collections.map(col => col.models).flat(); + return {collection: behavior.model.collection(allModels), type: behavior.type}; })); log.info(`Finished fetching entities from database for types ${entityBehaviors.map(({type}) => type).join(', ')}`); if (allEntities || entityType === 'Work') { log.info('Attaching author names to Work entities'); const authorCollection = entityLists.find( - (result) => result.model instanceof Author - ); - const workCollection = entityLists.find((result) => result.model instanceof Work); + (result) => result.type === 'Author' + )?.collection; + const workCollection = entityLists.find((result) => result.type === 'Work')?.collection; workCollection?.forEach((workEntity) => { const relationshipSet = workEntity.related('relationshipSet'); if (relationshipSet) { @@ -529,9 +529,8 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt const authorNames = []; authorWroteWorkRels.forEach((relationshipModel) => { // Search for the Author in the already fetched BookshelfJS Collection - const source = authorCollection.get( - relationshipModel.get('sourceBbid') - ); + const sourceBBID = relationshipModel.get('sourceBbid'); + const source = authorCollection.get(sourceBBID); const name = source?.related('defaultAlias')?.get('name'); if (name) { authorNames.push(name); @@ -545,7 +544,7 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt const listIndexes = []; // Index all the entities entityLists.forEach((entityList) => { - const listArray = entityList.map(getDocumentToIndex); + const listArray = entityList.collection.map(getDocumentToIndex); listIndexes.push(_processEntityListForBulk(listArray)); }); From 465f635f80a71a7a1a6c3e1b56eaf77828e46448 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 4 Jun 2024 19:11:31 +0200 Subject: [PATCH 10/14] Refactor entity fetching Parallelize promises --- src/common/helpers/search.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/helpers/search.ts b/src/common/helpers/search.ts index 912b1ea12..8dccc0007 100644 --- a/src/common/helpers/search.ts +++ b/src/common/helpers/search.ts @@ -487,24 +487,24 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt }).count(); log.info(`${totalCount} ${behavior.type} models in total`); const maxChunk = 50000; - const collections = []; + const collectionsPromises = []; // Fetch by chunks of 50.000 entities for (let i = 0; i < totalCount; i += maxChunk) { - log.info(`Fetching ${behavior.type} models with offset ${i}`); - // eslint-disable-next-line no-await-in-loop - const collection = await behavior.model + const collection = behavior.model .forge() .query((qb) => { qb.where('master', true); qb.whereNotNull('data_id'); qb.limit(maxChunk); qb.offset(i); + log.info(`Fetching ${maxChunk} ${behavior.type} models with offset ${i}`); }) .fetchAll({ withRelated: baseRelations.concat(behavior.relations) }).catch(err => { log.error(err); throw err; }); - collections.push(collection); + collectionsPromises.push(collection); } + const collections = await Promise.all(collectionsPromises); // Put all models back into a single collection const allModels = collections.map(col => col.models).flat(); return {collection: behavior.model.collection(allModels), type: behavior.type}; From c9658a4bb149c25079b8d41335f663ce11845812 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 4 Jun 2024 19:33:22 +0200 Subject: [PATCH 11/14] Fix search test --- test/src/api/routes/test-search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/api/routes/test-search.js b/test/src/api/routes/test-search.js index 2100ef2f2..589e254e8 100644 --- a/test/src/api/routes/test-search.js +++ b/test/src/api/routes/test-search.js @@ -45,7 +45,7 @@ describe('GET /search', () => { aliasData.sortName = 'Fnord'; await createEditionGroup(); // reindex elasticSearch - await search.generateIndex(orm); + await search.generateIndex(orm, 'allEntities', true); }); after(truncateEntities); From 837cfdb7592d6d812e0c29f58c19e7bd3b60f944 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Tue, 11 Jun 2024 18:25:58 +0200 Subject: [PATCH 12/14] Fix tests The few tests that use the generateIndex function needed to be updated --- test/src/server/routes/collection.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/src/server/routes/collection.js b/test/src/server/routes/collection.js index 91d20a1fc..5ba3993bd 100644 --- a/test/src/server/routes/collection.js +++ b/test/src/server/routes/collection.js @@ -26,7 +26,7 @@ describe('POST /collection/create', () => { } // The `agent` now has the sessionid cookie saved, and will send it // back to the server in the next request: - await generateIndex(orm); + await generateIndex(orm, 'Collection', true); agent = await chai.request.agent(app); await agent.get('/cb'); }); @@ -206,7 +206,7 @@ describe('POST collection/edit', () => { const res = await agent.post('/collection/create/handler').send(data); const collection = await new UserCollection({id: res.body.id}).fetch({withRelated: ['collaborators']}); collectionJSON = collection.toJSON(); - await generateIndex(orm); + await generateIndex(orm, 'Collection', true); }); after((done) => { // Clear DB tables then close superagent server @@ -463,7 +463,7 @@ describe('POST /collection/collectionID/delete', () => { // The `agent` now has the sessionid cookie saved, and will send it // back to the server in the next request: agent = await chai.request.agent(app); - await generateIndex(orm); + await generateIndex(orm, 'Collection', true); await agent.get('/cb'); }); after((done) => { @@ -482,7 +482,7 @@ describe('POST /collection/collectionID/delete', () => { public: true }; const collection = await new UserCollection(collectionData).save(null, {method: 'insert'}); - await generateIndex(orm); + await generateIndex(orm, 'Collection'); const res = await agent.post(`/collection/${collection.get('id')}/delete/handler`).send(); const collections = await new UserCollection().where('id', collection.get('id')).fetchAll({require: false}); const collectionsJSON = collections.toJSON(); @@ -500,7 +500,7 @@ describe('POST /collection/collectionID/delete', () => { public: true }; const collection = await new UserCollection(collectionData).save(null, {method: 'insert'}); - await generateIndex(orm); + await generateIndex(orm, 'Collection'); const author1 = await createAuthor(); const author2 = await createAuthor(); await new UserCollectionItem({ @@ -534,7 +534,7 @@ describe('POST /collection/collectionID/delete', () => { const collaborator1 = await createEditor(); const collaborator2 = await createEditor(); const collection = await new UserCollection(collectionData).save(null, {method: 'insert'}); - await generateIndex(orm); + await generateIndex(orm, 'Collection'); await new UserCollectionCollaborator({ collaboratorId: collaborator1.get('id'), collectionId: collection.get('id') @@ -566,7 +566,7 @@ describe('POST /collection/collectionID/delete', () => { }; const collection = await new UserCollection(collectionData).save(null, {method: 'insert'}); - await generateIndex(orm); + await generateIndex(orm, 'Collection'); // here loggedInUser is neither owner nor collaborator const response = await agent.post(`/collection/${collection.get('id')}/delete/handler`).send(); const collections = await new UserCollection().where('id', collection.get('id')).fetchAll({require: false}); @@ -587,7 +587,7 @@ describe('POST /collection/collectionID/delete', () => { public: true }; const collection = await new UserCollection(collectionData).save(null, {method: 'insert'}); - await generateIndex(orm); + await generateIndex(orm, 'Collection'); await new UserCollectionCollaborator({ collaboratorId: loggedInUser.get('id'), collectionId: collection.get('id') @@ -611,7 +611,7 @@ describe('POST /collection/collectionID/delete', () => { public: true }; const collection = await new UserCollection(collectionData).save(null, {method: 'insert'}); - await generateIndex(orm); + await generateIndex(orm, 'Collection'); const oldResult = await searchByName(orm, collectionData.name, 'Collection', 10, 0); const res = await agent.post(`/collection/${collection.get('id')}/delete/handler`).send(); await new Promise(resolve => setTimeout(resolve, 500)); @@ -1189,7 +1189,7 @@ describe('POST /collection/collectionID/collaborator/remove', () => { // The `agent` now has the sessionid cookie saved, and will send it // back to the server in the next request: agent = await chai.request.agent(app); - await generateIndex(orm); + await generateIndex(orm, 'Collection', true); await agent.get('/cb'); }); after((done) => { From b185de2c4d92573e910703fa53b6763bf18cce18 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Wed, 19 Jun 2024 15:09:14 +0200 Subject: [PATCH 13/14] Search: Only delete main search index if it exists Otherwise indices.delete will throw an error --- src/common/helpers/search.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/helpers/search.ts b/src/common/helpers/search.ts index 8dccc0007..ee65e02ca 100644 --- a/src/common/helpers/search.ts +++ b/src/common/helpers/search.ts @@ -417,8 +417,10 @@ export async function generateIndex(orm, entityType: IndexableEntities | 'allEnt const shouldRecreateIndex = !mainIndexExists.body || recreateIndex || allEntities; if (shouldRecreateIndex) { - log.notice('Deleting search index'); - await _client.indices.delete({index: _index}); + if (mainIndexExists.body) { + log.notice('Deleting search index'); + await _client.indices.delete({index: _index}); + } log.notice('Creating new search index'); await _client.indices.create({body: indexMappings, index: _index}); } From 2b312929e809fb04999b3fc20f9ba63dba783695 Mon Sep 17 00:00:00 2001 From: Monkey Do Date: Wed, 19 Jun 2024 16:14:40 +0200 Subject: [PATCH 14/14] Search: Fix optional search type If the type is not defined in the search page query, the move to using String(req.query.type) returned the string 'undefined' instead of defaulting to 'allEntities' --- src/server/routes/search.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/routes/search.tsx b/src/server/routes/search.tsx index 44353e2c2..a18f6355f 100644 --- a/src/server/routes/search.tsx +++ b/src/server/routes/search.tsx @@ -47,7 +47,7 @@ const {REINDEX_SEARCH_SERVER} = PrivilegeType; router.get('/', async (req, res, next) => { const {orm} = req.app.locals; const query = req.query.q ?? ''; - const type = String(req.query.type) || 'allEntities'; + const type = req.query.type?.toString() || 'allEntities'; const size = req.query.size ? parseInt(String(req.query.size), 10) : 20; const from = req.query.from ? parseInt(String(req.query.from), 10) : 0; try { @@ -121,7 +121,7 @@ router.get('/search', (req, res) => { router.get('/autocomplete', (req, res) => { const {orm} = req.app.locals; const query = req.query.q; - const type = req.query.type || 'allEntities'; + const type = req.query.type?.toString() || 'allEntities'; const size = Number(req.query.size) || 42; const searchPromise = search.autocomplete(orm, query, type, size);