diff --git a/.docker/aws-resources/elasticsearch/esindex_corpus_de.json b/.docker/aws-resources/elasticsearch/esindex_corpus_de.json index 6ed34e87c..7a0503140 100644 --- a/.docker/aws-resources/elasticsearch/esindex_corpus_de.json +++ b/.docker/aws-resources/elasticsearch/esindex_corpus_de.json @@ -2,7 +2,7 @@ "settings": { "index": { "number_of_shards": 1, - "number_of_replicas": 1, + "number_of_replicas": 0, "knn": true } }, diff --git a/.docker/aws-resources/elasticsearch/esindex_corpus_en_luc.json b/.docker/aws-resources/elasticsearch/esindex_corpus_en_luc.json index 8998d7f4a..c6e526393 100644 --- a/.docker/aws-resources/elasticsearch/esindex_corpus_en_luc.json +++ b/.docker/aws-resources/elasticsearch/esindex_corpus_en_luc.json @@ -1,8 +1,8 @@ { "settings": { "index": { - "number_of_shards": 2, - "number_of_replicas": 1, + "number_of_shards": 1, + "number_of_replicas": 0, "knn": true } }, diff --git a/.docker/aws-resources/elasticsearch/esindex_corpus_es.json b/.docker/aws-resources/elasticsearch/esindex_corpus_es.json index 26192270b..193c4703b 100644 --- a/.docker/aws-resources/elasticsearch/esindex_corpus_es.json +++ b/.docker/aws-resources/elasticsearch/esindex_corpus_es.json @@ -2,7 +2,7 @@ "settings": { "index": { "number_of_shards": 1, - "number_of_replicas": 1, + "number_of_replicas": 0, "knn": true } }, diff --git a/.docker/aws-resources/elasticsearch/esindex_corpus_fr.json b/.docker/aws-resources/elasticsearch/esindex_corpus_fr.json index 07d33de3d..90810d7bc 100644 --- a/.docker/aws-resources/elasticsearch/esindex_corpus_fr.json +++ b/.docker/aws-resources/elasticsearch/esindex_corpus_fr.json @@ -2,7 +2,7 @@ "settings": { "index": { "number_of_shards": 1, - "number_of_replicas": 1, + "number_of_replicas": 0, "knn": true } }, diff --git a/.docker/aws-resources/elasticsearch/esindex_corpus_it.json b/.docker/aws-resources/elasticsearch/esindex_corpus_it.json index cb7b1404a..a94b0194e 100644 --- a/.docker/aws-resources/elasticsearch/esindex_corpus_it.json +++ b/.docker/aws-resources/elasticsearch/esindex_corpus_it.json @@ -2,7 +2,7 @@ "settings": { "index": { "number_of_shards": 1, - "number_of_replicas": 1, + "number_of_replicas": 0, "knn": true } }, diff --git a/.docker/aws-resources/elasticsearch/esindex_list.json b/.docker/aws-resources/elasticsearch/esindex_list.json index f42a39761..a50333fca 100755 --- a/.docker/aws-resources/elasticsearch/esindex_list.json +++ b/.docker/aws-resources/elasticsearch/esindex_list.json @@ -1,8 +1,8 @@ { "settings": { "index": { - "number_of_shards": 10, - "number_of_replicas": 2 + "number_of_shards": 1, + "number_of_replicas": 0 } }, "mappings": { diff --git a/docker-compose.yml b/docker-compose.yml index fcff6f12c..08787fa96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,7 +64,7 @@ services: start_period: 80s localstack: - image: localstack/localstack:3.7.1@sha256:7f4f51bbfcfb42c661b9c02716a3cfcb53433f3c066905329aa911178835b300 + image: localstack/localstack:4.0.2 ports: - '4566:4566' volumes: diff --git a/servers/user-list-search/src/corpus/keywordSearch.integration.ts b/servers/user-list-search/src/corpus/keywordSearch.integration.ts index 011c34877..2cfff1111 100644 --- a/servers/user-list-search/src/corpus/keywordSearch.integration.ts +++ b/servers/user-list-search/src/corpus/keywordSearch.integration.ts @@ -23,7 +23,7 @@ describe('Corpus search - keyword', () => { beforeAll(async () => { await deleteDocuments(); await seedCorpus(); - await unleash( + unleash( mockFlags([ { name: config.unleash.flags.semanticSearch.name, enabled: false }, ]), diff --git a/servers/user-list-search/src/corpus/semanticSearch.integration.ts b/servers/user-list-search/src/corpus/semanticSearch.integration.ts index 7eeacd355..f8d973929 100644 --- a/servers/user-list-search/src/corpus/semanticSearch.integration.ts +++ b/servers/user-list-search/src/corpus/semanticSearch.integration.ts @@ -32,7 +32,7 @@ describe('Corpus search - semantic', () => { const clientMock: any = jest.spyOn(Client.prototype, 'search'); beforeAll(async () => { - await unleash( + unleash( mockFlags([ { name: config.unleash.flags.semanticSearch.name, enabled: true }, ]), diff --git a/servers/user-list-search/src/datasource/elasticsearch/advancedSearch.integration.ts b/servers/user-list-search/src/datasource/elasticsearch/advancedSearch.integration.ts index abb27251f..4b22de294 100644 --- a/servers/user-list-search/src/datasource/elasticsearch/advancedSearch.integration.ts +++ b/servers/user-list-search/src/datasource/elasticsearch/advancedSearch.integration.ts @@ -1,7 +1,5 @@ import { advancedSearch } from './elasticsearchSearch'; import { bulkDocument } from './elasticsearchBulk'; -import { client } from './index'; -import { config } from '../../config'; import { faker } from '@faker-js/faker'; import { SearchItemsContentType, @@ -9,6 +7,9 @@ import { SearchItemsSortOrder, SearchItemsStatusFilter, } from '../../__generated__/types'; +import { deleteDocuments } from '../../test/utils/searchIntegrationTestHelpers'; +import { client } from '.'; +import { config } from '../../config'; const defaultDocProps = { resolved_id: 1, @@ -25,20 +26,7 @@ const defaultDocProps = { describe('Elasticsearch Search Query', () => { beforeEach(async () => { - await client.deleteByQuery({ - index: config.aws.elasticsearch.list.index, - body: { - query: { - match_all: {}, - }, - }, - }); - - // Wait for delete to finish - await client.indices.refresh({ - index: config.aws.elasticsearch.list.index, - }); - + await deleteDocuments(); await bulkDocument([ { ...defaultDocProps, @@ -100,9 +88,14 @@ describe('Elasticsearch Search Query', () => { }, ]); - // wait for 1 second. I noticed test wasn't passing if run took quickly after inserting. There may be more to do - // here to make sure this does not become brittle. - await new Promise((resolve) => setTimeout(resolve, 1000)); + // Wait for index to finish + await client.indices.refresh({ + index: config.aws.elasticsearch.list.index, + }); + }); + + afterAll(async () => { + await deleteDocuments(); }); it.each([ { diff --git a/servers/user-list-search/src/datasource/elasticsearch/elasticsearchBulk.integration.ts b/servers/user-list-search/src/datasource/elasticsearch/elasticsearchBulk.integration.ts index 965338cfd..88b0e16db 100644 --- a/servers/user-list-search/src/datasource/elasticsearch/elasticsearchBulk.integration.ts +++ b/servers/user-list-search/src/datasource/elasticsearch/elasticsearchBulk.integration.ts @@ -3,6 +3,7 @@ import { client } from './index'; import { bulkDocument } from './elasticsearchBulk'; import { getDocument } from './elasticsearchSearch'; +import { deleteDocuments } from '../../test/utils/searchIntegrationTestHelpers'; const defaultDocProps = { resolved_id: 1, @@ -19,19 +20,7 @@ const defaultDocProps = { describe('Elasticsearch Bulk', () => { beforeAll(async () => { - await client.deleteByQuery({ - index: config.aws.elasticsearch.list.index, - body: { - query: { - match_all: {}, - }, - }, - }); - - // Wait for delete to finish - await client.indices.refresh({ - index: config.aws.elasticsearch.list.index, - }); + await deleteDocuments(); await bulkDocument([ { @@ -51,6 +40,10 @@ describe('Elasticsearch Bulk', () => { }); }); + afterAll(async () => { + await deleteDocuments(); + }); + it('can bulk index a document', async () => { const document = (await getDocument('1-12345')) as any; expect(document).not.toBeNull(); diff --git a/servers/user-list-search/src/datasource/elasticsearch/elasticsearchSearch.integration.ts b/servers/user-list-search/src/datasource/elasticsearch/elasticsearchSearch.integration.ts index 4fe830d83..5cf758f50 100644 --- a/servers/user-list-search/src/datasource/elasticsearch/elasticsearchSearch.integration.ts +++ b/servers/user-list-search/src/datasource/elasticsearch/elasticsearchSearch.integration.ts @@ -8,6 +8,7 @@ import { } from '../../__generated__/types'; import { client } from './index'; import { config } from '../../config'; +import { deleteDocuments } from '../../test/utils/searchIntegrationTestHelpers'; const defaultDocProps = { resolved_id: 1, @@ -23,19 +24,7 @@ const defaultDocProps = { describe('Elasticsearch Search Query', () => { beforeEach(async () => { - await client.deleteByQuery({ - index: config.aws.elasticsearch.list.index, - body: { - query: { - match_all: {}, - }, - }, - }); - - // Wait for delete to finish - await client.indices.refresh({ - index: config.aws.elasticsearch.list.index, - }); + await deleteDocuments(); await bulkDocument([ { @@ -139,6 +128,10 @@ describe('Elasticsearch Search Query', () => { }); }); + afterAll(async () => { + await deleteDocuments(); + }); + // For now this gives us confidence that basic search is not // broken when we make updates it('can search for a document', async () => { diff --git a/servers/user-list-search/src/saves/elasticsearch.integration.ts b/servers/user-list-search/src/saves/elasticsearch.integration.ts index 5c45b9da6..f035393ff 100644 --- a/servers/user-list-search/src/saves/elasticsearch.integration.ts +++ b/servers/user-list-search/src/saves/elasticsearch.integration.ts @@ -2,6 +2,7 @@ import { client } from '../datasource/elasticsearch'; import { config } from '../config'; import { bulkDocument } from '../datasource/elasticsearch/elasticsearchBulk'; import { deleteSearchIndexByUserId } from './elasticsearch'; +import { deleteDocuments } from '../test/utils/searchIntegrationTestHelpers'; const defaultDocProps = { resolved_id: 1, @@ -17,19 +18,7 @@ const defaultDocProps = { describe('Elasticsearch - Integration', () => { beforeEach(async () => { - await client.deleteByQuery({ - index: config.aws.elasticsearch.list.index, - body: { - query: { - match_all: {}, - }, - }, - }); - - // Wait for delete to finish - await client.indices.refresh({ - index: config.aws.elasticsearch.list.index, - }); + await deleteDocuments(); await bulkDocument([ { diff --git a/servers/user-list-search/src/saves/premiumSearch.integration.ts b/servers/user-list-search/src/saves/premiumSearch.integration.ts index a326684d7..486d86e4c 100644 --- a/servers/user-list-search/src/saves/premiumSearch.integration.ts +++ b/servers/user-list-search/src/saves/premiumSearch.integration.ts @@ -12,6 +12,7 @@ import { } from '../datasource/clients/knexClient'; import { Knex } from 'knex'; import { + deleteDocuments, loadItemExtended, loadList, } from '../test/utils/searchIntegrationTestHelpers'; @@ -76,7 +77,6 @@ async function loadUserItem( } describe('premium search functional test', () => { - const testEsClient = client; let app: Application; let server: ApolloServer; let url: string; @@ -129,20 +129,7 @@ describe('premium search functional test', () => { beforeAll(async () => { ({ app, server, url } = await startServer(0)); - await testEsClient.deleteByQuery({ - index: config.aws.elasticsearch.list.index, - type: config.aws.elasticsearch.list.type, - body: { - query: { - match_all: {}, - }, - }, - }); - - // Wait for delete to finish - await testEsClient.indices.refresh({ - index: config.aws.elasticsearch.list.index, - }); + await deleteDocuments(); await db('readitla_ril-tmp.list').truncate(); await db('readitla_b.items_extended').truncate(); @@ -236,7 +223,7 @@ describe('premium search functional test', () => { await Promise.all(items); // Takes a hot sec for the data to be available, otherwise test flakes - await testEsClient.indices.refresh({ + await client.indices.refresh({ index: config.aws.elasticsearch.list.index, }); }); @@ -245,7 +232,8 @@ describe('premium search functional test', () => { await server.stop(); await db.destroy(); await knexDbWriteClient().destroy(); - testEsClient.close(); + await deleteDocuments(); + client.close(); }); it('should search for Items under User entity', async () => { diff --git a/servers/user-list-search/src/saves/premiumSearchByOffset.integration.ts b/servers/user-list-search/src/saves/premiumSearchByOffset.integration.ts index aeeb7d50b..f54a8bb93 100644 --- a/servers/user-list-search/src/saves/premiumSearchByOffset.integration.ts +++ b/servers/user-list-search/src/saves/premiumSearchByOffset.integration.ts @@ -12,6 +12,7 @@ import { } from '../datasource/clients/knexClient'; import { Knex } from 'knex'; import { + deleteDocuments, loadItemExtended, loadList, } from '../test/utils/searchIntegrationTestHelpers'; @@ -78,7 +79,6 @@ async function loadUserItem( } describe('premium search functional test (offset pagination)', () => { - const testEsClient = client; let app: Application; let server: ApolloServer; let url: string; @@ -124,19 +124,7 @@ describe('premium search functional test (offset pagination)', () => { beforeAll(async () => { ({ app, server, url } = await startServer(0)); - await testEsClient.deleteByQuery({ - index: config.aws.elasticsearch.list.index, - body: { - query: { - match_all: {}, - }, - }, - }); - - // Wait for delete to finish - await testEsClient.indices.refresh({ - index: config.aws.elasticsearch.list.index, - }); + await deleteDocuments(); await db('readitla_ril-tmp.list').truncate(); await db('readitla_b.items_extended').truncate(); @@ -228,7 +216,7 @@ describe('premium search functional test (offset pagination)', () => { ]; await Promise.all(items); // Takes a hot sec for the data to be available, otherwise test flakes - await testEsClient.indices.refresh({ + await client.indices.refresh({ index: config.aws.elasticsearch.list.index, }); }); @@ -237,7 +225,8 @@ describe('premium search functional test (offset pagination)', () => { await server.stop(); await db.destroy(); await knexDbWriteClient().destroy(); - testEsClient.close(); + await deleteDocuments(); + client.close(); }); it('should handle response that returns hits with no highlights object', async () => { diff --git a/servers/user-list-search/src/server/routes/batchDelete.ts b/servers/user-list-search/src/server/routes/batchDelete.ts index acf2c5dfa..6c853f624 100644 --- a/servers/user-list-search/src/server/routes/batchDelete.ts +++ b/servers/user-list-search/src/server/routes/batchDelete.ts @@ -29,10 +29,7 @@ export function validate( ): Response { const errors = validationResult(req); if (!errors.isEmpty()) { - return res - .status(400) - .json({ errors: errors.array() }) - .setHeader('Content-Type', 'application/json'); + return res.status(400).json({ errors: errors.array() }); } next(); } diff --git a/servers/user-list-search/src/test/utils/corpusSeeder.ts b/servers/user-list-search/src/test/utils/corpusSeeder.ts index e26a173f1..de33f207d 100644 --- a/servers/user-list-search/src/test/utils/corpusSeeder.ts +++ b/servers/user-list-search/src/test/utils/corpusSeeder.ts @@ -49,6 +49,11 @@ export async function deleteDocuments() { body: { query: { match_all: {} } }, wait_for_completion: true, }); + // Forcemerge the index because with our updates and number of documents the score can change when it should not. + // https://kulekci.medium.com/understanding-and-resolving-elasticsearch-score-changes-after-document-updates-a9f426b76e38 + await client.indices.forcemerge({ + index, + }); await client.indices.refresh({ index, }); diff --git a/servers/user-list-search/src/test/utils/searchIntegrationTestHelpers.ts b/servers/user-list-search/src/test/utils/searchIntegrationTestHelpers.ts index f599d35d7..7cf47ab6e 100644 --- a/servers/user-list-search/src/test/utils/searchIntegrationTestHelpers.ts +++ b/servers/user-list-search/src/test/utils/searchIntegrationTestHelpers.ts @@ -1,4 +1,6 @@ import { Knex } from 'knex'; +import { client } from '../../datasource/elasticsearch'; +import { config } from '../../config'; export type SeedData = { favorite: number; @@ -13,6 +15,37 @@ export type SeedData = { isVideo?: number; }; +/** + * Test cleanup: delete all documents in search indices + */ +export async function deleteDocuments() { + try { + await client.deleteByQuery({ + index: config.aws.elasticsearch.list.index, + body: { + query: { + match_all: {}, + }, + }, + waitForCompletion: true, + conflicts: 'proceed', + }); + + // Forcemerge the index because with our updates and number of documents the score can change when it should not. + // https://kulekci.medium.com/understanding-and-resolving-elasticsearch-score-changes-after-document-updates-a9f426b76e38 + await client.indices.forcemerge({ + index: config.aws.elasticsearch.list.index, + }); + + // Wait for delete to finish + await client.indices.refresh({ + index: config.aws.elasticsearch.list.index, + }); + } catch (error) { + console.log(error); + } +} + export async function loadItemExtended(db, seedData: SeedData) { const optionalDefaults = { lang: 'en',