Skip to content

Commit 0e1b2a3

Browse files
hop-devhop-dev
authored
[Entity Store] [FTR Tests] Fix flakiness + poll for engine started on setup (elastic#196564)
## Summary Closes elastic#196546 Closes elastic#196526 Unskips flaky entity store tests after fixes. Entity store tests were not polling for the engine to be started before asserting the assets were present. I have also added some retries to the asset checks as some assets are not immediately queryable after creation.
1 parent 10ec204 commit 0e1b2a3

File tree

6 files changed

+151
-82
lines changed

6 files changed

+151
-82
lines changed

‎x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts

+28-2
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,36 @@ export const executeFieldRetentionEnrichPolicy = async ({
7272
export const deleteFieldRetentionEnrichPolicy = async ({
7373
unitedDefinition,
7474
esClient,
75+
logger,
76+
attempts = 5,
77+
delayMs = 2000,
7578
}: {
76-
esClient: ElasticsearchClient;
7779
unitedDefinition: DefinitionMetadata;
80+
esClient: ElasticsearchClient;
81+
logger: Logger;
82+
attempts?: number;
83+
delayMs?: number;
7884
}) => {
7985
const name = getFieldRetentionEnrichPolicyName(unitedDefinition);
80-
return esClient.enrich.deletePolicy({ name }, { ignore: [404] });
86+
let currentAttempt = 1;
87+
while (currentAttempt <= attempts) {
88+
try {
89+
await esClient.enrich.deletePolicy({ name }, { ignore: [404] });
90+
return;
91+
} catch (e) {
92+
// a 429 status code indicates that the enrich policy is being executed
93+
if (currentAttempt === attempts || e.statusCode !== 429) {
94+
logger.error(
95+
`Error deleting enrich policy ${name}: ${e.message} after ${currentAttempt} attempts`
96+
);
97+
throw e;
98+
}
99+
100+
logger.info(
101+
`Enrich policy ${name} is being executed, waiting for it to finish before deleting`
102+
);
103+
await new Promise((resolve) => setTimeout(resolve, delayMs));
104+
currentAttempt++;
105+
}
106+
}
81107
};

‎x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ export class EntityStoreDataClient {
245245
logger,
246246
taskManager,
247247
});
248-
logger.info(`Entity store initialized`);
248+
logger.info(`Entity store initialized for ${entityType}`);
249249

250250
return updated;
251251
} catch (err) {
@@ -362,6 +362,7 @@ export class EntityStoreDataClient {
362362
await deleteFieldRetentionEnrichPolicy({
363363
unitedDefinition,
364364
esClient: this.esClient,
365+
logger,
365366
});
366367

367368
if (deleteData) {
@@ -450,7 +451,7 @@ export class EntityStoreDataClient {
450451
originalStatus === ENGINE_STATUS.UPDATING
451452
) {
452453
throw new Error(
453-
`Error updating entity store: There is an changes already in progress for engine ${id}`
454+
`Error updating entity store: There are changes already in progress for engine ${id}`
454455
);
455456
}
456457

‎x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts

+8-12
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,7 @@ export default ({ getService }: FtrProviderContext) => {
1414
const supertest = getService('supertest');
1515

1616
const utils = EntityStoreUtils(getService);
17-
// Failing: See https://github.com/elastic/kibana/issues/196526
18-
describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => {
17+
describe('@ess @skipInServerlessMKI Entity Store Engine APIs', () => {
1918
const dataView = dataViewRouteHelpersFactory(supertest);
2019

2120
before(async () => {
@@ -33,22 +32,19 @@ export default ({ getService }: FtrProviderContext) => {
3332
});
3433

3534
it('should have installed the expected user resources', async () => {
36-
await utils.initEntityEngineForEntityType('user');
35+
await utils.initEntityEngineForEntityTypesAndWait(['user']);
3736
await utils.expectEngineAssetsExist('user');
3837
});
3938

4039
it('should have installed the expected host resources', async () => {
41-
await utils.initEntityEngineForEntityType('host');
40+
await utils.initEntityEngineForEntityTypesAndWait(['host']);
4241
await utils.expectEngineAssetsExist('host');
4342
});
4443
});
4544

4645
describe('get and list', () => {
4746
before(async () => {
48-
await Promise.all([
49-
utils.initEntityEngineForEntityType('host'),
50-
utils.initEntityEngineForEntityType('user'),
51-
]);
47+
await utils.initEntityEngineForEntityTypesAndWait(['host', 'user']);
5248
});
5349

5450
after(async () => {
@@ -118,7 +114,7 @@ export default ({ getService }: FtrProviderContext) => {
118114

119115
describe('start and stop', () => {
120116
before(async () => {
121-
await utils.initEntityEngineForEntityType('host');
117+
await utils.initEntityEngineForEntityTypesAndWait(['host']);
122118
});
123119

124120
after(async () => {
@@ -160,7 +156,7 @@ export default ({ getService }: FtrProviderContext) => {
160156

161157
describe('delete', () => {
162158
it('should delete the host entity engine', async () => {
163-
await utils.initEntityEngineForEntityType('host');
159+
await utils.initEntityEngineForEntityTypesAndWait(['host']);
164160

165161
await api
166162
.deleteEntityEngine({
@@ -173,7 +169,7 @@ export default ({ getService }: FtrProviderContext) => {
173169
});
174170

175171
it('should delete the user entity engine', async () => {
176-
await utils.initEntityEngineForEntityType('user');
172+
await utils.initEntityEngineForEntityTypesAndWait(['user']);
177173

178174
await api
179175
.deleteEntityEngine({
@@ -188,7 +184,7 @@ export default ({ getService }: FtrProviderContext) => {
188184

189185
describe('apply_dataview_indices', () => {
190186
before(async () => {
191-
await utils.initEntityEngineForEntityType('host');
187+
await utils.initEntityEngineForEntityTypesAndWait(['host']);
192188
});
193189

194190
after(async () => {

‎x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts

+7-11
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
1818
const supertest = getService('supertest');
1919
const utils = EntityStoreUtils(getService, namespace);
2020

21-
// Failing: See https://github.com/elastic/kibana/issues/196546
22-
describe.skip('@ess Entity Store Engine APIs in non-default space', () => {
21+
describe('@ess Entity Store Engine APIs in non-default space', () => {
2322
const dataView = dataViewRouteHelpersFactory(supertest, namespace);
2423

2524
before(async () => {
@@ -43,22 +42,19 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
4342
});
4443

4544
it('should have installed the expected user resources', async () => {
46-
await utils.initEntityEngineForEntityType('user');
45+
await utils.initEntityEngineForEntityTypesAndWait(['user']);
4746
await utils.expectEngineAssetsExist('user');
4847
});
4948

5049
it('should have installed the expected host resources', async () => {
51-
await utils.initEntityEngineForEntityType('host');
50+
await utils.initEntityEngineForEntityTypesAndWait(['host']);
5251
await utils.expectEngineAssetsExist('host');
5352
});
5453
});
5554

5655
describe('get and list', () => {
5756
before(async () => {
58-
await Promise.all([
59-
utils.initEntityEngineForEntityType('host'),
60-
utils.initEntityEngineForEntityType('user'),
61-
]);
57+
await utils.initEntityEngineForEntityTypesAndWait(['host', 'user']);
6258
});
6359

6460
after(async () => {
@@ -134,7 +130,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
134130

135131
describe('start and stop', () => {
136132
before(async () => {
137-
await utils.initEntityEngineForEntityType('host');
133+
await utils.initEntityEngineForEntityTypesAndWait(['host']);
138134
});
139135

140136
after(async () => {
@@ -188,7 +184,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
188184

189185
describe('delete', () => {
190186
it('should delete the host entity engine', async () => {
191-
await utils.initEntityEngineForEntityType('host');
187+
await utils.initEntityEngineForEntityTypesAndWait(['host']);
192188

193189
await api
194190
.deleteEntityEngine(
@@ -204,7 +200,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
204200
});
205201

206202
it('should delete the user entity engine', async () => {
207-
await utils.initEntityEngineForEntityType('user');
203+
await utils.initEntityEngineForEntityTypesAndWait(['user']);
208204

209205
await api
210206
.deleteEntityEngine(

‎x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts

+76-51
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import { FtrProviderContext } from '@kbn/ftr-common-functional-services';
99

1010
export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getService']) => {
1111
const es = getService('es');
12+
const retry = getService('retry');
13+
const log = getService('log');
1214

1315
const expectTransformExists = async (transformId: string) => {
1416
return expectTransformStatus(transformId, true);
@@ -18,45 +20,43 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe
1820
return expectTransformStatus(transformId, false);
1921
};
2022

21-
const expectTransformStatus = async (
22-
transformId: string,
23-
exists: boolean,
24-
attempts: number = 5,
25-
delayMs: number = 2000
26-
) => {
27-
let currentAttempt = 1;
28-
while (currentAttempt <= attempts) {
29-
try {
30-
await es.transform.getTransform({ transform_id: transformId });
31-
if (!exists) {
32-
throw new Error(`Expected transform ${transformId} to not exist, but it does`);
23+
const expectTransformStatus = async (transformId: string, exists: boolean) => {
24+
await retry.waitForWithTimeout(
25+
`transform ${transformId} to ${exists ? 'exist' : 'not exist'}`,
26+
10_000,
27+
async () => {
28+
try {
29+
await es.transform.getTransform({ transform_id: transformId });
30+
return exists;
31+
} catch (e) {
32+
log.debug(`Transform ${transformId} not found: ${e}`);
33+
return !exists;
3334
}
34-
return; // Transform exists, exit the loop
35-
} catch (e) {
36-
if (currentAttempt === attempts) {
37-
if (exists) {
38-
throw new Error(`Expected transform ${transformId} to exist, but it does not: ${e}`);
39-
} else {
40-
return; // Transform does not exist, exit the loop
41-
}
42-
}
43-
await new Promise((resolve) => setTimeout(resolve, delayMs));
44-
currentAttempt++;
4535
}
46-
}
36+
);
4737
};
4838

4939
const expectEnrichPolicyStatus = async (policyId: string, exists: boolean) => {
50-
try {
51-
await es.enrich.getPolicy({ name: policyId });
52-
if (!exists) {
53-
throw new Error(`Expected enrich policy ${policyId} to not exist, but it does`);
54-
}
55-
} catch (e) {
56-
if (exists) {
57-
throw new Error(`Expected enrich policy ${policyId} to exist, but it does not: ${e}`);
40+
await retry.waitForWithTimeout(
41+
`enrich policy ${policyId} to ${exists ? 'exist' : 'not exist'}`,
42+
20_000,
43+
async () => {
44+
try {
45+
const res = await es.enrich.getPolicy({ name: policyId });
46+
const policy = res.policies?.[0];
47+
if (policy) {
48+
log.debug(`Enrich policy ${policyId} found: ${JSON.stringify(res)}`);
49+
return exists;
50+
} else {
51+
log.debug(`Enrich policy ${policyId} not found: ${JSON.stringify(res)}`);
52+
return !exists;
53+
}
54+
} catch (e) {
55+
log.debug(`Enrich policy ${policyId} not found: ${e}`);
56+
return !exists;
57+
}
5858
}
59-
}
59+
);
6060
};
6161

6262
const expectEnrichPolicyExists = async (policyId: string) =>
@@ -66,18 +66,19 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe
6666
expectEnrichPolicyStatus(policyId, false);
6767

6868
const expectComponentTemplatStatus = async (templateName: string, exists: boolean) => {
69-
try {
70-
await es.cluster.getComponentTemplate({ name: templateName });
71-
if (!exists) {
72-
throw new Error(`Expected component template ${templateName} to not exist, but it does`);
73-
}
74-
} catch (e) {
75-
if (exists) {
76-
throw new Error(
77-
`Expected component template ${templateName} to exist, but it does not: ${e}`
78-
);
69+
await retry.waitForWithTimeout(
70+
`component template ${templateName} to ${exists ? 'exist' : 'not exist'}`,
71+
10_000,
72+
async () => {
73+
try {
74+
await es.cluster.getComponentTemplate({ name: templateName });
75+
return exists; // Component template exists
76+
} catch (e) {
77+
log.debug(`Component template ${templateName} not found: ${e}`);
78+
return !exists; // Component template does not exist
79+
}
7980
}
80-
}
81+
);
8182
};
8283

8384
const expectComponentTemplateExists = async (templateName: string) =>
@@ -87,23 +88,45 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe
8788
expectComponentTemplatStatus(templateName, false);
8889

8990
const expectIngestPipelineStatus = async (pipelineId: string, exists: boolean) => {
91+
await retry.waitForWithTimeout(
92+
`ingest pipeline ${pipelineId} to ${exists ? 'exist' : 'not exist'}`,
93+
10_000,
94+
async () => {
95+
try {
96+
await es.ingest.getPipeline({ id: pipelineId });
97+
return exists; // Ingest pipeline exists
98+
} catch (e) {
99+
log.debug(`Ingest pipeline ${pipelineId} not found: ${e}`);
100+
return !exists; // Ingest pipeline does not exist
101+
}
102+
}
103+
);
104+
};
105+
106+
const expectIngestPipelineExists = async (pipelineId: string) =>
107+
expectIngestPipelineStatus(pipelineId, true);
108+
109+
const expectIngestPipelineNotFound = async (pipelineId: string) =>
110+
expectIngestPipelineStatus(pipelineId, false);
111+
112+
const expectIndexStatus = async (indexName: string, exists: boolean) => {
90113
try {
91-
await es.ingest.getPipeline({ id: pipelineId });
114+
await es.indices.get({ index: indexName });
92115
if (!exists) {
93-
throw new Error(`Expected ingest pipeline ${pipelineId} to not exist, but it does`);
116+
throw new Error(`Expected index ${indexName} to not exist, but it does`);
94117
}
95118
} catch (e) {
96119
if (exists) {
97-
throw new Error(`Expected ingest pipeline ${pipelineId} to exist, but it does not: ${e}`);
120+
throw new Error(`Expected index ${indexName} to exist, but it does not: ${e}`);
98121
}
99122
}
100123
};
101124

102-
const expectIngestPipelineExists = async (pipelineId: string) =>
103-
expectIngestPipelineStatus(pipelineId, true);
125+
const expectEntitiesIndexExists = async (entityType: string, namespace: string) =>
126+
expectIndexStatus(`.entities.v1.latest.security_${entityType}_${namespace}`, true);
104127

105-
const expectIngestPipelineNotFound = async (pipelineId: string) =>
106-
expectIngestPipelineStatus(pipelineId, false);
128+
const expectEntitiesIndexNotFound = async (entityType: string, namespace: string) =>
129+
expectIndexStatus(`.entities.v1.latest.security_${entityType}_${namespace}`, false);
107130

108131
return {
109132
expectComponentTemplateExists,
@@ -112,6 +135,8 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe
112135
expectEnrichPolicyNotFound,
113136
expectIngestPipelineExists,
114137
expectIngestPipelineNotFound,
138+
expectEntitiesIndexExists,
139+
expectEntitiesIndexNotFound,
115140
expectTransformExists,
116141
expectTransformNotFound,
117142
};

0 commit comments

Comments
 (0)