[Entity Analytics] [Entity Store] [API] Changes to support event.ingested as a configurable timestamp field for init and enable endpoints

oas_docs/output/kibana.serverless.yaml

@@ -9826,6 +9826,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp.
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -9966,6 +9970,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp for the entity type.
+                  type: string
         description: Schema for the engine initialization
         required: true
       responses:
@@ -9983,7 +9991,7 @@ paths:
     post:
       operationId: StartEntityEngine
       parameters:
-        - description: The entity type of the engine (either 'user' or 'host').
+        - description: The entity type of the engine
           in: path
           name: entityType
           required: true
@@ -51430,6 +51438,8 @@ components:
           default: 180s
           pattern: '[smdh]$'
           type: string
+        timestampField:
+          type: string
         type:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
       required:
@@ -51557,6 +51567,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         host:
           type: object
           properties:
@@ -51595,7 +51611,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - host
         - entity
     Security_Entity_Analytics_API_IdField:
@@ -51702,6 +51717,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         service:
           type: object
           properties:
@@ -51712,7 +51733,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - service
         - entity
     Security_Entity_Analytics_API_StoreStatus:
@@ -51758,6 +51778,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         user:
           type: object
           properties:
@@ -51792,7 +51818,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - user
         - entity
     Security_Exceptions_API_CreateExceptionListItemComment:

oas_docs/output/kibana.yaml

@@ -11916,6 +11916,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp.
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -12052,6 +12056,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp for the entity type.
+                  type: string
         description: Schema for the engine initialization
         required: true
       responses:
@@ -12068,7 +12076,7 @@ paths:
     post:
       operationId: StartEntityEngine
       parameters:
-        - description: The entity type of the engine (either 'user' or 'host').
+        - description: The entity type of the engine
           in: path
           name: entityType
           required: true
@@ -58120,6 +58128,8 @@ components:
           default: 180s
           pattern: '[smdh]$'
           type: string
+        timestampField:
+          type: string
         type:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
       required:
@@ -58247,6 +58257,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         host:
           type: object
           properties:
@@ -58285,7 +58301,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - host
         - entity
     Security_Entity_Analytics_API_IdField:
@@ -58392,6 +58407,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         service:
           type: object
           properties:
@@ -58402,7 +58423,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - service
         - entity
     Security_Entity_Analytics_API_StoreStatus:
@@ -58448,6 +58468,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         user:
           type: object
           properties:
@@ -58482,7 +58508,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - user
         - entity
     Security_Exceptions_API_CreateExceptionListItemComment:

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -321,6 +321,7 @@
     "filter",
     "indexPattern",
     "status",
+    "timestampField",
     "type"
   ],
   "epm-packages": [

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1092,6 +1092,9 @@
       "status": {
         "type": "keyword"
       },
+      "timestampField": {
+        "type": "keyword"
+      },
       "type": {
         "type": "keyword"
       }

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -97,7 +97,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
         "entity-definition": "1c6bff35c423d5dc5650bc806cf2899e4706a0bc",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
-        "entity-engine-status": "e2de87d84e9f1f72726eb28b7e670ff8021b5eb4",
+        "entity-engine-status": "09f6a617020708e4f638137e5ef35bd9534133be",
         "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
         "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts

@@ -41,6 +41,7 @@ export const EngineDescriptor = z.object({
     .regex(/[smdh]$/)
     .optional()
     .default('24h'),
+  timestampField: z.string().optional(),
   timeout: z
     .string()
     .regex(/[smdh]$/)

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml

@@ -36,6 +36,8 @@ components:
           type: string
           default: 24h
           pattern: '[smdh]$'
+        timestampField:
+          type: string
         timeout:
           type: string
           default: 180s

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts

@@ -28,6 +28,10 @@ export const InitEntityStoreRequestBody = z.object({
   filter: z.string().optional(),
   entityTypes: z.array(EntityType).optional(),
   enrichPolicyExecutionInterval: Interval.optional(),
+  /**
+   * The field to use as the timestamp.
+   */
+  timestampField: z.string().optional().default('@timestamp'),
   /**
    * The amount of time the transform looks back to calculate the aggregations.
    */

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml

@@ -33,6 +33,10 @@ paths:
                     $ref: './common.schema.yaml#/components/schemas/EntityType'
                 enrichPolicyExecutionInterval:
                   $ref: './common.schema.yaml#/components/schemas/Interval'
+                timestampField:
+                  type: string
+                  description: The field to use as the timestamp.
+                  default: '@timestamp'
                 lookbackPeriod: 
                   type: string
                   default: 24h

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.gen.ts

@@ -36,6 +36,10 @@ export const InitEntityEngineRequestBody = z.object({
   indexPattern: IndexPattern.optional(),
   filter: z.string().optional(),
   enrichPolicyExecutionInterval: Interval.optional(),
+  /**
+   * The field to use as the timestamp for the entity type.
+   */
+  timestampField: z.string().optional().default('@timestamp'),
   /**
    * The amount of time the transform looks back to calculate the aggregations.
    */

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml

@@ -35,7 +35,10 @@ paths:
                   type: string
                 enrichPolicyExecutionInterval:
                   $ref: '../common.schema.yaml#/components/schemas/Interval'
-
+                timestampField:
+                  type: string
+                  description: The field to use as the timestamp for the entity type.
+                  default: '@timestamp'
                 lookbackPeriod: 
                   type: string
                   default: 24h

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/start.gen.ts

@@ -21,7 +21,7 @@ import { EntityType } from '../common.gen';
 export type StartEntityEngineRequestParams = z.infer<typeof StartEntityEngineRequestParams>;
 export const StartEntityEngineRequestParams = z.object({
   /**
-   * The entity type of the engine (either 'user' or 'host').
+   * The entity type of the engine
    */
   entityType: EntityType,
 });

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/start.schema.yaml

@@ -16,7 +16,7 @@ paths:
           required: true
           schema:
             $ref: '../common.schema.yaml#/components/schemas/EntityType'
-          description: The entity type of the engine (either 'user' or 'host').
+          description: The entity type of the engine
       responses:
         '200':
           description: Successful response

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/entities/common.gen.ts

@@ -21,7 +21,7 @@ import { AssetCriticalityLevel } from '../../asset_criticality/common.gen';
 
 export type UserEntity = z.infer<typeof UserEntity>;
 export const UserEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -41,11 +41,16 @@ export const UserEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export type HostEntity = z.infer<typeof HostEntity>;
 export const HostEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -66,11 +71,16 @@ export const HostEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export type ServiceEntity = z.infer<typeof ServiceEntity>;
 export const ServiceEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -84,6 +94,11 @@ export const ServiceEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export const EntityInternal = z.union([UserEntity, HostEntity, ServiceEntity]);