[SecuritySolution] Add enrichPolicyExecutionInterval to entity enable…

…ment and init APIs (#207374) ## Summary Add `enrichPolicyExecutionInterval`param to entity enablement and init APIs ### How to test? * Start kibana * Call the entity store enablement API with a short value for `enrichPolicyExecutionInterval` param * Check in the logs if the enrichment process is running frequently * Clear the entity store * Call the entity store enablement API without `enrichPolicyExecutionInterval` param * Check in the logs if the enrichment process is running less frequently **Enable Entity store API call:** ``` POST kbn:/api/entity_store/enable { "enrichPolicyExecutionInterval": "10s" } ``` **Init Entity store API call:** ``` POST kbn:/api/entity_store/engines/user/init { "enrichPolicyExecutionInterval": "10s" } ``` **Enrich policy log message:** ``` │ info [o.e.x.e.EnrichPolicyRunner] [...] Policy [entity_store_field_retention_user_default_v1.0.0]: Running enrich policy ``` ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Jan 24, 2025 · 1ca4d96 · 1ca4d96
1 parent 3805b46
commit 1ca4d96
Show file tree

Hide file tree

Showing 13 changed files with 81 additions and 8 deletions.
diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml
@@ -9721,6 +9721,8 @@ paths:
             schema:
               type: object
               properties:
+                enrichPolicyExecutionInterval:
+                  $ref: '#/components/schemas/Security_Entity_Analytics_API_Interval'
                 entityTypes:
                   items:
                     $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
@@ -9840,6 +9842,8 @@ paths:
             schema:
               type: object
               properties:
+                enrichPolicyExecutionInterval:
+                  $ref: '#/components/schemas/Security_Entity_Analytics_API_Interval'
                 fieldHistoryLength:
                   default: 10
                   description: The number of historical values to keep for each field.
@@ -50543,6 +50547,11 @@ components:
       required:
         - dsl
         - response
+    Security_Entity_Analytics_API_Interval:
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      example: 1h
+      pattern: ^[1-9]\d*[smh]$
+      type: string
     Security_Entity_Analytics_API_RiskEngineScheduleNowErrorResponse:
       type: object
       properties:

diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml
@@ -11892,6 +11892,8 @@ paths:
             schema:
               type: object
               properties:
+                enrichPolicyExecutionInterval:
+                  $ref: '#/components/schemas/Security_Entity_Analytics_API_Interval'
                 entityTypes:
                   items:
                     $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
@@ -12007,6 +12009,8 @@ paths:
             schema:
               type: object
               properties:
+                enrichPolicyExecutionInterval:
+                  $ref: '#/components/schemas/Security_Entity_Analytics_API_Interval'
                 fieldHistoryLength:
                   default: 10
                   description: The number of historical values to keep for each field.
@@ -57418,6 +57422,11 @@ components:
       required:
         - dsl
         - response
+    Security_Entity_Analytics_API_Interval:
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      example: 1h
+      pattern: ^[1-9]\d*[smh]$
+      type: string
     Security_Entity_Analytics_API_RiskEngineScheduleNowErrorResponse:
       type: object
       properties:

diff --git a/...security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/...security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
@@ -80,3 +80,9 @@ export const InspectQuery = z.object({
   response: z.array(z.string()),
   dsl: z.array(z.string()),
 });
+
+/**
+ * Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+ */
+export type Interval = z.infer<typeof Interval>;
+export const Interval = z.string().regex(/^[1-9]\d*[smh]$/);
diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
@@ -113,3 +113,8 @@ components:
       required:
         - dsl
         - response
+    Interval:
+      type: string
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      pattern: '^[1-9]\d*[smh]$' # any number except zero followed by one of the suffixes 's', 'm', 'h'
+      example: '1h'
diff --git a/...security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts b/...security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts
@@ -16,7 +16,7 @@
 
 import { z } from '@kbn/zod';
 
-import { IndexPattern, EntityType, EngineDescriptor } from './common.gen';
+import { IndexPattern, EntityType, Interval, EngineDescriptor } from './common.gen';
 
 export type InitEntityStoreRequestBody = z.infer<typeof InitEntityStoreRequestBody>;
 export const InitEntityStoreRequestBody = z.object({
@@ -27,6 +27,7 @@ export const InitEntityStoreRequestBody = z.object({
   indexPattern: IndexPattern.optional(),
   filter: z.string().optional(),
   entityTypes: z.array(EntityType).optional(),
+  enrichPolicyExecutionInterval: Interval.optional(),
 });
 export type InitEntityStoreRequestBodyInput = z.input<typeof InitEntityStoreRequestBody>;
 

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml
@@ -31,6 +31,8 @@ paths:
                   type: array
                   items:
                     $ref: './common.schema.yaml#/components/schemas/EntityType'
+                enrichPolicyExecutionInterval:
+                  $ref: './common.schema.yaml#/components/schemas/Interval'
       responses:
         '200':
           description: Successful response

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.gen.ts b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.gen.ts
@@ -16,7 +16,7 @@
 
 import { z } from '@kbn/zod';
 
-import { EntityType, IndexPattern, EngineDescriptor } from '../common.gen';
+import { EntityType, IndexPattern, Interval, EngineDescriptor } from '../common.gen';
 
 export type InitEntityEngineRequestParams = z.infer<typeof InitEntityEngineRequestParams>;
 export const InitEntityEngineRequestParams = z.object({
@@ -35,6 +35,7 @@ export const InitEntityEngineRequestBody = z.object({
   fieldHistoryLength: z.number().int().optional().default(10),
   indexPattern: IndexPattern.optional(),
   filter: z.string().optional(),
+  enrichPolicyExecutionInterval: Interval.optional(),
 });
 export type InitEntityEngineRequestBodyInput = z.input<typeof InitEntityEngineRequestBody>;
 

diff --git a/...lugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml b/...lugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml
@@ -33,6 +33,8 @@ paths:
                   $ref: '../common.schema.yaml#/components/schemas/IndexPattern'
                 filter:
                   type: string
+                enrichPolicyExecutionInterval:
+                  $ref: '../common.schema.yaml#/components/schemas/Interval'
       responses:
         '200':
           description: Successful response

diff --git a/...on/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/...on/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -307,6 +307,8 @@ paths:
             schema:
               type: object
               properties:
+                enrichPolicyExecutionInterval:
+                  $ref: '#/components/schemas/Interval'
                 entityTypes:
                   items:
                     $ref: '#/components/schemas/EntityType'
@@ -422,6 +424,8 @@ paths:
             schema:
               type: object
               properties:
+                enrichPolicyExecutionInterval:
+                  $ref: '#/components/schemas/Interval'
                 fieldHistoryLength:
                   default: 10
                   description: The number of historical values to keep for each field.
@@ -1211,6 +1215,13 @@ components:
       required:
         - dsl
         - response
+    Interval:
+      description: >-
+        Interval in which enrich policy runs. For example, `"1h"` means the rule
+        runs every hour.
+      example: 1h
+      pattern: ^[1-9]\d*[smh]$
+      type: string
     RiskEngineScheduleNowErrorResponse:
       type: object
       properties:

diff --git a/.../openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/.../openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -307,6 +307,8 @@ paths:
             schema:
               type: object
               properties:
+                enrichPolicyExecutionInterval:
+                  $ref: '#/components/schemas/Interval'
                 entityTypes:
                   items:
                     $ref: '#/components/schemas/EntityType'
@@ -422,6 +424,8 @@ paths:
             schema:
               type: object
               properties:
+                enrichPolicyExecutionInterval:
+                  $ref: '#/components/schemas/Interval'
                 fieldHistoryLength:
                   default: 10
                   description: The number of historical values to keep for each field.
@@ -1211,6 +1215,13 @@ components:
       required:
         - dsl
         - response
+    Interval:
+      description: >-
+        Interval in which enrich policy runs. For example, `"1h"` means the rule
+        runs every hour.
+      example: 1h
+      pattern: ^[1-9]\d*[smh]$
+      type: string
     RiskEngineScheduleNowErrorResponse:
       type: object
       properties:

diff --git a/...ns/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/...ns/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
@@ -91,6 +91,7 @@ import {
   createKeywordBuilderPipeline,
   deleteKeywordBuilderPipeline,
 } from '../../asset_inventory/ingest_pipelines';
+import { DEFAULT_INTERVAL } from './task/constants';
 
 // Workaround. TransformState type is wrong. The health type should be: TransformHealth from '@kbn/transform-plugin/common/types/transform_stats'
 export interface TransformHealth extends estypes.TransformGetTransformStatsTransformStatsHealth {
@@ -208,6 +209,7 @@ export class EntityStoreDataClient {
       filter = '',
       fieldHistoryLength = 10,
       entityTypes,
+      enrichPolicyExecutionInterval,
     }: InitEntityStoreRequestBody,
     { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
   ): Promise<InitEntityStoreResponse> {
@@ -229,7 +231,11 @@ export class EntityStoreDataClient {
 
     const promises = enginesTypes.map((entity) =>
       run(() =>
-        this.init(entity, { indexPattern, filter, fieldHistoryLength }, { pipelineDebugMode })
+        this.init(
+          entity,
+          { indexPattern, filter, fieldHistoryLength, enrichPolicyExecutionInterval },
+          { pipelineDebugMode }
+        )
       )
     );
 
@@ -287,7 +293,12 @@ export class EntityStoreDataClient {
 
   public async init(
     entityType: EntityType,
-    { indexPattern = '', filter = '', fieldHistoryLength = 10 }: InitEntityEngineRequestBody,
+    {
+      indexPattern = '',
+      filter = '',
+      fieldHistoryLength = 10,
+      enrichPolicyExecutionInterval = DEFAULT_INTERVAL,
+    }: InitEntityEngineRequestBody,
     { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
   ): Promise<InitEntityEngineResponse> {
     const { experimentalFeatures } = this.options;
@@ -344,6 +355,7 @@ export class EntityStoreDataClient {
     this.asyncSetup(
       entityType,
       fieldHistoryLength,
+      enrichPolicyExecutionInterval,
       this.options.taskManager,
       indexPattern,
       filter,
@@ -359,6 +371,7 @@ export class EntityStoreDataClient {
   private async asyncSetup(
     entityType: EntityType,
     fieldHistoryLength: number,
+    enrichPolicyExecutionInterval: string,
     taskManager: TaskManagerStartContract,
     indexPattern: string,
     filter: string,
@@ -447,6 +460,7 @@ export class EntityStoreDataClient {
         namespace,
         logger,
         taskManager,
+        interval: enrichPolicyExecutionInterval,
       });
 
       this.log(`debug`, entityType, `Started entity store field retention enrich task`);

diff --git a/...rity/plugins/security_solution/server/lib/entity_analytics/entity_store/task/constants.ts b/...rity/plugins/security_solution/server/lib/entity_analytics/entity_store/task/constants.ts
@@ -8,5 +8,5 @@
 export const SCOPE = ['securitySolution'];
 export const TYPE = 'entity_store:field_retention:enrichment';
 export const VERSION = '1.0.0';
-export const INTERVAL = '1h';
+export const DEFAULT_INTERVAL = '1h';
 export const TIMEOUT = '10m';
diff --git a/...solution/server/lib/entity_analytics/entity_store/task/field_retention_enrichment_task.ts b/...solution/server/lib/entity_analytics/entity_store/task/field_retention_enrichment_task.ts
@@ -24,7 +24,7 @@ import {
   stateSchemaByVersion,
   type LatestTaskStateSchema as EntityStoreFieldRetentionTaskState,
 } from './state';
-import { INTERVAL, SCOPE, TIMEOUT, TYPE, VERSION } from './constants';
+import { SCOPE, TIMEOUT, TYPE, VERSION } from './constants';
 import type { EntityAnalyticsRoutesDeps } from '../../types';
 
 import { executeFieldRetentionEnrichPolicy } from '../elasticsearch_assets';
@@ -120,10 +120,12 @@ export const startEntityStoreFieldRetentionEnrichTask = async ({
   logger,
   namespace,
   taskManager,
+  interval,
 }: {
   logger: Logger;
   namespace: string;
   taskManager: TaskManagerStartContract;
+  interval: string;
 }) => {
   const taskId = getTaskId(namespace);
   const log = logFactory(logger, taskId);
@@ -136,7 +138,7 @@ export const startEntityStoreFieldRetentionEnrichTask = async ({
       taskType: getTaskName(),
       scope: SCOPE,
       schedule: {
-        interval: INTERVAL,
+        interval,
       },
       state: { ...defaultState, namespace },
       params: { version: VERSION },
@@ -234,7 +236,7 @@ export const runTask = async ({
 
     telemetry.reportEvent(FIELD_RETENTION_ENRICH_POLICY_EXECUTION_EVENT.eventType, {
       duration: taskDurationInSeconds,
-      interval: INTERVAL,
+      interval: taskInstance.schedule?.interval,
     });
 
     // Track entity store usage