From d65c7e6cf2d7a6a7a37d5c37642e1dc9c04c8c11 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Fri, 28 Jun 2024 08:02:54 +0200 Subject: [PATCH] fix: improve disabling and restoration of ES indexing settings (#4187) --- .../src/migrations/5.38.0/002/ddb-es/index.ts | 1 - .../src/migrations/5.39.0/001/ddb-es/index.ts | 1 - .../src/migrations/5.39.2/001/ddb-es/index.ts | 1 - .../src/migrations/5.39.6/001/ddb-es/index.ts | 1 - .../src/utils/elasticsearch/disableEsIndexing.ts | 1 - .../elasticsearch/fetchOriginalEsSettings.ts | 4 +--- .../elasticsearch/restoreOriginalEsSettings.ts | 16 ++++++++++++---- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/migrations/src/migrations/5.38.0/002/ddb-es/index.ts b/packages/migrations/src/migrations/5.38.0/002/ddb-es/index.ts index b2e365be06f..09c05f68d3d 100644 --- a/packages/migrations/src/migrations/5.38.0/002/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.38.0/002/ddb-es/index.ts @@ -35,7 +35,6 @@ interface LastEvaluatedKey { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } diff --git a/packages/migrations/src/migrations/5.39.0/001/ddb-es/index.ts b/packages/migrations/src/migrations/5.39.0/001/ddb-es/index.ts index 8ce9267f12a..eae8ee791f4 100644 --- a/packages/migrations/src/migrations/5.39.0/001/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.39.0/001/ddb-es/index.ts @@ -39,7 +39,6 @@ interface LastEvaluatedKey { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } diff --git a/packages/migrations/src/migrations/5.39.2/001/ddb-es/index.ts b/packages/migrations/src/migrations/5.39.2/001/ddb-es/index.ts index 93a61137087..3cb23aeebea 100644 --- a/packages/migrations/src/migrations/5.39.2/001/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.39.2/001/ddb-es/index.ts @@ -43,7 +43,6 @@ interface LastEvaluatedKey { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/index.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/index.ts index fdf0511cd89..392c76a254e 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/index.ts @@ -46,7 +46,6 @@ interface LastEvaluatedKey { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } diff --git a/packages/migrations/src/utils/elasticsearch/disableEsIndexing.ts b/packages/migrations/src/utils/elasticsearch/disableEsIndexing.ts index 13984d373fd..48d014bf695 100644 --- a/packages/migrations/src/utils/elasticsearch/disableEsIndexing.ts +++ b/packages/migrations/src/utils/elasticsearch/disableEsIndexing.ts @@ -18,7 +18,6 @@ export const disableElasticsearchIndexing = async ( elasticsearchClient: params.elasticsearchClient, index, settings: { - number_of_replicas: 0, refresh_interval: -1 } }); diff --git a/packages/migrations/src/utils/elasticsearch/fetchOriginalEsSettings.ts b/packages/migrations/src/utils/elasticsearch/fetchOriginalEsSettings.ts index 5a6f0ca6601..0c277910189 100644 --- a/packages/migrations/src/utils/elasticsearch/fetchOriginalEsSettings.ts +++ b/packages/migrations/src/utils/elasticsearch/fetchOriginalEsSettings.ts @@ -9,7 +9,6 @@ interface FetchOriginalElasticsearchSettingsParams { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } @@ -21,10 +20,9 @@ export const fetchOriginalElasticsearchSettings = async ( const settings = await esGetIndexSettings({ elasticsearchClient: params.elasticsearchClient, index, - fields: ["number_of_replicas", "refresh_interval"] + fields: ["refresh_interval"] }); return { - number_of_replicas: settings.number_of_replicas || 1, refresh_interval: settings.refresh_interval || "1s" }; } catch (ex) { diff --git a/packages/migrations/src/utils/elasticsearch/restoreOriginalEsSettings.ts b/packages/migrations/src/utils/elasticsearch/restoreOriginalEsSettings.ts index bfbe59bd6f3..2d321d6c740 100644 --- a/packages/migrations/src/utils/elasticsearch/restoreOriginalEsSettings.ts +++ b/packages/migrations/src/utils/elasticsearch/restoreOriginalEsSettings.ts @@ -3,8 +3,7 @@ import { Logger } from "@webiny/data-migration"; import { Client } from "@elastic/elasticsearch"; interface IndexSettings { - number_of_replicas: number; - refresh_interval: `${number}s`; + refresh_interval: `${number}s` | "-1"; } interface RestoreOriginalElasticsearchSettingsParams { @@ -30,13 +29,22 @@ export const restoreOriginalElasticsearchSettings = async ( if (!settings || typeof settings !== "object") { continue; } + + // We must ensure that the refresh interval is not set to a negative value. Why? + // We've had a case where a migration run has been manually stopped, and the index settings + // were never restored. Once a second run was started and this restore function + // was called, the refresh interval was set to `-1s`, which effectively disabled indexing. + let refreshInterval = settings.refresh_interval || `1s`; + if (refreshInterval === "-1") { + refreshInterval = "1s"; + } + try { await esPutIndexSettings({ elasticsearchClient: params.elasticsearchClient, index, settings: { - number_of_replicas: settings.number_of_replicas || 1, - refresh_interval: settings.refresh_interval || `1s` + refresh_interval: refreshInterval } }); } catch (ex) {