[8.x] [Entity Analytics] [Entity Store] [API] Changes to support even…

…t.ingested as a configurable timestamp field for init and enable endpoints (#208201) (#208866)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Entity Analytics] [Entity Store] [API] Changes to support
event.ingested as a configurable timestamp field for init and enable
endpoints (#208201)](#208201)

<!--- Backport version: 9.6.4 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Abhishek
Bhatia","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-01-29T23:49:13Z","message":"[Entity
Analytics] [Entity Store] [API] Changes to support event.ingested as a
configurable timestamp field for init and enable endpoints
(#208201)\n\n## Summary\n\nThis PR introduces support for configuring
the `timestamp` field for\nentity store enablement.\n\nBy default, the
`timestamp` field is set to `@timestamp`, but users can\nopt to use
`event.ingested` or another preferred value based on
their\nrequirements.\n\n\n### Entity Store API changes\n\n#### Entity
Store enable\n\n```\nPOST kbn:/api/entity_store/enable\n{\n
\"timestampField\": \"event.ingested\"\n}\n```\n\n#### Result
\n\n```\n{\n \"engines\": [\n {\n \"status\": \"started\",\n \"type\":
\"user\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"event.ingested\"\n
},\n {\n \"status\": \"started\",\n \"type\": \"host\",\n
\"indexPattern\": \"\",\n \"filter\": \"\",\n \"fieldHistoryLength\":
10,\n \"timestampField\": \"event.ingested\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"universal\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"event.ingested\"\n }\n ],\n \"status\":
\"running\"\n}\n```\n\n```\nPOST
kbn:/api/entity_store/enable\n{\n}\n```\n#### Result\n\n```\n{\n
\"engines\": [\n {\n \"status\": \"started\",\n \"type\": \"host\",\n
\"indexPattern\": \"\",\n \"filter\": \"\",\n \"fieldHistoryLength\":
10,\n \"timestampField\": \"@timestamp\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"user\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"@timestamp\"\n },\n {\n \"status\": \"started\",\n \"type\":
\"universal\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"@timestamp\"\n }\n
],\n \"status\": \"running\"\n}\n```\n\nDifferent entity types can have
distinct `timestampField` values, as\ndescribed below:\n\nFor instance,
the `host` entity uses `event.ingested` as its\n`timestampField`, while
other entities default to the `@timestamp`\nfield.\n\n```\nPOST
kbn:/api/entity_store/engines/host/init\n{\n \"timestampField\":
\"event.ingested\"\n}\n\nPOST
kbn:/api/entity_store/engines/user/init\n{\n}\n```\n\n####
Result\n\n```\n{\n \"engines\": [\n {\n \"status\": \"started\",\n
\"type\": \"universal\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"@timestamp\"\n },\n
{\n \"status\": \"started\",\n \"type\": \"host\",\n \"indexPattern\":
\"\",\n \"filter\": \"\",\n \"fieldHistoryLength\": 10,\n
\"timestampField\": \"event.ingested\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"user\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"@timestamp\"\n }\n ],\n \"status\": \"running\"\n}\n```\n\n\n###
Checklist\n\nCheck the PR satisfies following conditions. \n\nReviewers
should verify this PR satisfies this list as well.\n\n- [x] This was
checked for breaking HTTP API changes, and any breaking\nchanges have
been approved by the breaking-change committee.
The\n`release_note:breaking` label should be applied in these
situations.\n- [x] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests changed\n- [x] The PR description includes the
appropriate Release Notes section,\nand the correct `release_note:*`
label is applied per
the\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)\n\n\n###
Testing steps\n\n- Check out this PR branch.\n- Start Kibana.\n- Avoid
enabling the Entity Store from the UI initially. Instead, enable\nit via
the API by following the steps outlined below.\n\n```\nDELETE
kbn:/api/entity_store/engines/user\nDELETE
kbn:/api/entity_store/engines/host\n\nGET
kbn:/api/entity_store/status\n\n All engines with the same timestamp
Field\nPOST kbn:/api/entity_store/enable\n{\n \"timestampField\":
\"event.ingested\"\n}\n\n# All engines with the same timestamp
Field\nPOST kbn:/api/entity_store/enable\n{\n}\n\n# Different entity
with different timestamp Field\nPOST
kbn:/api/entity_store/engines/host/init\n{\n \"timestampField\":
\"event.ingested\"\n}\n\nPOST
kbn:/api/entity_store/engines/user/init\n{\n}\n```\n\n4. Afterward, you
can test the functionality through the UI. First,\nclear the entity data
from the UI, then start the Entity Store. Note\nthat the UI currently
does not offer a way to configure this
parameter.\n\n---------\n\nCo-authored-by: kibanamachine
<[email protected]>\nCo-authored-by:
Elastic Machine
<[email protected]>\nCo-authored-by: Mark Hopkin
<[email protected]>","sha":"a060baee83f9be2402cb1a48b0ff2126d8938b56","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:enhancement","v9.0.0","Team:Entity
Analytics","backport:version","v8.18.0"],"title":"[Entity Analytics]
[Entity Store] [API] Changes to support event.ingested as a configurable
timestamp field for init and enable
endpoints","number":208201,"url":"https://github.com/elastic/kibana/pull/208201","mergeCommit":{"message":"[Entity
Analytics] [Entity Store] [API] Changes to support event.ingested as a
configurable timestamp field for init and enable endpoints
(#208201)\n\n## Summary\n\nThis PR introduces support for configuring
the `timestamp` field for\nentity store enablement.\n\nBy default, the
`timestamp` field is set to `@timestamp`, but users can\nopt to use
`event.ingested` or another preferred value based on
their\nrequirements.\n\n\n### Entity Store API changes\n\n#### Entity
Store enable\n\n```\nPOST kbn:/api/entity_store/enable\n{\n
\"timestampField\": \"event.ingested\"\n}\n```\n\n#### Result
\n\n```\n{\n \"engines\": [\n {\n \"status\": \"started\",\n \"type\":
\"user\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"event.ingested\"\n
},\n {\n \"status\": \"started\",\n \"type\": \"host\",\n
\"indexPattern\": \"\",\n \"filter\": \"\",\n \"fieldHistoryLength\":
10,\n \"timestampField\": \"event.ingested\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"universal\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"event.ingested\"\n }\n ],\n \"status\":
\"running\"\n}\n```\n\n```\nPOST
kbn:/api/entity_store/enable\n{\n}\n```\n#### Result\n\n```\n{\n
\"engines\": [\n {\n \"status\": \"started\",\n \"type\": \"host\",\n
\"indexPattern\": \"\",\n \"filter\": \"\",\n \"fieldHistoryLength\":
10,\n \"timestampField\": \"@timestamp\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"user\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"@timestamp\"\n },\n {\n \"status\": \"started\",\n \"type\":
\"universal\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"@timestamp\"\n }\n
],\n \"status\": \"running\"\n}\n```\n\nDifferent entity types can have
distinct `timestampField` values, as\ndescribed below:\n\nFor instance,
the `host` entity uses `event.ingested` as its\n`timestampField`, while
other entities default to the `@timestamp`\nfield.\n\n```\nPOST
kbn:/api/entity_store/engines/host/init\n{\n \"timestampField\":
\"event.ingested\"\n}\n\nPOST
kbn:/api/entity_store/engines/user/init\n{\n}\n```\n\n####
Result\n\n```\n{\n \"engines\": [\n {\n \"status\": \"started\",\n
\"type\": \"universal\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"@timestamp\"\n },\n
{\n \"status\": \"started\",\n \"type\": \"host\",\n \"indexPattern\":
\"\",\n \"filter\": \"\",\n \"fieldHistoryLength\": 10,\n
\"timestampField\": \"event.ingested\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"user\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"@timestamp\"\n }\n ],\n \"status\": \"running\"\n}\n```\n\n\n###
Checklist\n\nCheck the PR satisfies following conditions. \n\nReviewers
should verify this PR satisfies this list as well.\n\n- [x] This was
checked for breaking HTTP API changes, and any breaking\nchanges have
been approved by the breaking-change committee.
The\n`release_note:breaking` label should be applied in these
situations.\n- [x] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests changed\n- [x] The PR description includes the
appropriate Release Notes section,\nand the correct `release_note:*`
label is applied per
the\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)\n\n\n###
Testing steps\n\n- Check out this PR branch.\n- Start Kibana.\n- Avoid
enabling the Entity Store from the UI initially. Instead, enable\nit via
the API by following the steps outlined below.\n\n```\nDELETE
kbn:/api/entity_store/engines/user\nDELETE
kbn:/api/entity_store/engines/host\n\nGET
kbn:/api/entity_store/status\n\n All engines with the same timestamp
Field\nPOST kbn:/api/entity_store/enable\n{\n \"timestampField\":
\"event.ingested\"\n}\n\n# All engines with the same timestamp
Field\nPOST kbn:/api/entity_store/enable\n{\n}\n\n# Different entity
with different timestamp Field\nPOST
kbn:/api/entity_store/engines/host/init\n{\n \"timestampField\":
\"event.ingested\"\n}\n\nPOST
kbn:/api/entity_store/engines/user/init\n{\n}\n```\n\n4. Afterward, you
can test the functionality through the UI. First,\nclear the entity data
from the UI, then start the Entity Store. Note\nthat the UI currently
does not offer a way to configure this
parameter.\n\n---------\n\nCo-authored-by: kibanamachine
<[email protected]>\nCo-authored-by:
Elastic Machine
<[email protected]>\nCo-authored-by: Mark Hopkin
<[email protected]>","sha":"a060baee83f9be2402cb1a48b0ff2126d8938b56"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/208201","number":208201,"mergeCommit":{"message":"[Entity
Analytics] [Entity Store] [API] Changes to support event.ingested as a
configurable timestamp field for init and enable endpoints
(#208201)\n\n## Summary\n\nThis PR introduces support for configuring
the `timestamp` field for\nentity store enablement.\n\nBy default, the
`timestamp` field is set to `@timestamp`, but users can\nopt to use
`event.ingested` or another preferred value based on
their\nrequirements.\n\n\n### Entity Store API changes\n\n#### Entity
Store enable\n\n```\nPOST kbn:/api/entity_store/enable\n{\n
\"timestampField\": \"event.ingested\"\n}\n```\n\n#### Result
\n\n```\n{\n \"engines\": [\n {\n \"status\": \"started\",\n \"type\":
\"user\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"event.ingested\"\n
},\n {\n \"status\": \"started\",\n \"type\": \"host\",\n
\"indexPattern\": \"\",\n \"filter\": \"\",\n \"fieldHistoryLength\":
10,\n \"timestampField\": \"event.ingested\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"universal\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"event.ingested\"\n }\n ],\n \"status\":
\"running\"\n}\n```\n\n```\nPOST
kbn:/api/entity_store/enable\n{\n}\n```\n#### Result\n\n```\n{\n
\"engines\": [\n {\n \"status\": \"started\",\n \"type\": \"host\",\n
\"indexPattern\": \"\",\n \"filter\": \"\",\n \"fieldHistoryLength\":
10,\n \"timestampField\": \"@timestamp\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"user\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"@timestamp\"\n },\n {\n \"status\": \"started\",\n \"type\":
\"universal\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"@timestamp\"\n }\n
],\n \"status\": \"running\"\n}\n```\n\nDifferent entity types can have
distinct `timestampField` values, as\ndescribed below:\n\nFor instance,
the `host` entity uses `event.ingested` as its\n`timestampField`, while
other entities default to the `@timestamp`\nfield.\n\n```\nPOST
kbn:/api/entity_store/engines/host/init\n{\n \"timestampField\":
\"event.ingested\"\n}\n\nPOST
kbn:/api/entity_store/engines/user/init\n{\n}\n```\n\n####
Result\n\n```\n{\n \"engines\": [\n {\n \"status\": \"started\",\n
\"type\": \"universal\",\n \"indexPattern\": \"\",\n \"filter\": \"\",\n
\"fieldHistoryLength\": 10,\n \"timestampField\": \"@timestamp\"\n },\n
{\n \"status\": \"started\",\n \"type\": \"host\",\n \"indexPattern\":
\"\",\n \"filter\": \"\",\n \"fieldHistoryLength\": 10,\n
\"timestampField\": \"event.ingested\"\n },\n {\n \"status\":
\"started\",\n \"type\": \"user\",\n \"indexPattern\": \"\",\n
\"filter\": \"\",\n \"fieldHistoryLength\": 10,\n \"timestampField\":
\"@timestamp\"\n }\n ],\n \"status\": \"running\"\n}\n```\n\n\n###
Checklist\n\nCheck the PR satisfies following conditions. \n\nReviewers
should verify this PR satisfies this list as well.\n\n- [x] This was
checked for breaking HTTP API changes, and any breaking\nchanges have
been approved by the breaking-change committee.
The\n`release_note:breaking` label should be applied in these
situations.\n- [x] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests changed\n- [x] The PR description includes the
appropriate Release Notes section,\nand the correct `release_note:*`
label is applied per
the\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)\n\n\n###
Testing steps\n\n- Check out this PR branch.\n- Start Kibana.\n- Avoid
enabling the Entity Store from the UI initially. Instead, enable\nit via
the API by following the steps outlined below.\n\n```\nDELETE
kbn:/api/entity_store/engines/user\nDELETE
kbn:/api/entity_store/engines/host\n\nGET
kbn:/api/entity_store/status\n\n All engines with the same timestamp
Field\nPOST kbn:/api/entity_store/enable\n{\n \"timestampField\":
\"event.ingested\"\n}\n\n# All engines with the same timestamp
Field\nPOST kbn:/api/entity_store/enable\n{\n}\n\n# Different entity
with different timestamp Field\nPOST
kbn:/api/entity_store/engines/host/init\n{\n \"timestampField\":
\"event.ingested\"\n}\n\nPOST
kbn:/api/entity_store/engines/user/init\n{\n}\n```\n\n4. Afterward, you
can test the functionality through the UI. First,\nclear the entity data
from the UI, then start the Entity Store. Note\nthat the UI currently
does not offer a way to configure this
parameter.\n\n---------\n\nCo-authored-by: kibanamachine
<[email protected]>\nCo-authored-by:
Elastic Machine
<[email protected]>\nCo-authored-by: Mark Hopkin
<[email protected]>","sha":"a060baee83f9be2402cb1a48b0ff2126d8938b56"}},{"branch":"8.x","label":"v8.18.0","branchLabelMappingKey":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Abhishek Bhatia <[email protected]>

oas_docs/output/kibana.serverless.yaml

@@ -7489,6 +7489,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp.
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -7629,6 +7633,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp for the entity type.
+                  type: string
         description: Schema for the engine initialization
         required: true
       responses:
@@ -7646,7 +7654,7 @@ paths:
     post:
       operationId: StartEntityEngine
       parameters:
-        - description: The entity type of the engine (either 'user' or 'host').
+        - description: The entity type of the engine
           in: path
           name: entityType
           required: true
@@ -47684,6 +47692,8 @@ components:
           default: 180s
           pattern: '[smdh]$'
           type: string
+        timestampField:
+          type: string
         type:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
       required:
@@ -47811,6 +47821,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         host:
           type: object
           properties:
@@ -47849,7 +47865,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - host
         - entity
     Security_Entity_Analytics_API_IdField:
@@ -47956,6 +47971,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         service:
           type: object
           properties:
@@ -47966,7 +47987,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - service
         - entity
     Security_Entity_Analytics_API_StoreStatus:
@@ -48012,6 +48032,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         user:
           type: object
           properties:
@@ -48046,7 +48072,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - user
         - entity
     Security_Exceptions_API_CreateExceptionListItemComment:

oas_docs/output/kibana.yaml

@@ -13070,6 +13070,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp.
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -13206,6 +13210,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp for the entity type.
+                  type: string
         description: Schema for the engine initialization
         required: true
       responses:
@@ -13222,7 +13230,7 @@ paths:
     post:
       operationId: StartEntityEngine
       parameters:
-        - description: The entity type of the engine (either 'user' or 'host').
+        - description: The entity type of the engine
           in: path
           name: entityType
           required: true
@@ -36075,6 +36083,8 @@ components:
           default: 180s
           pattern: '[smdh]$'
           type: string
+        timestampField:
+          type: string
         type:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
       required:
@@ -36202,6 +36212,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         host:
           type: object
           properties:
@@ -36240,7 +36256,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - host
         - entity
     Security_Entity_Analytics_API_IdField:
@@ -36347,6 +36362,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         service:
           type: object
           properties:
@@ -36357,7 +36378,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - service
         - entity
     Security_Entity_Analytics_API_StoreStatus:
@@ -36403,6 +36423,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         user:
           type: object
           properties:
@@ -36437,7 +36463,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - user
         - entity
     Security_Exceptions_API_CreateExceptionListItemComment:

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -321,6 +321,7 @@
     "filter",
     "indexPattern",
     "status",
+    "timestampField",
     "type"
   ],
   "epm-packages": [

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1092,6 +1092,9 @@
       "status": {
         "type": "keyword"
       },
+      "timestampField": {
+        "type": "keyword"
+      },
       "type": {
         "type": "keyword"
       }

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -94,7 +94,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
         "entity-definition": "1c6bff35c423d5dc5650bc806cf2899e4706a0bc",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
-        "entity-engine-status": "e2de87d84e9f1f72726eb28b7e670ff8021b5eb4",
+        "entity-engine-status": "09f6a617020708e4f638137e5ef35bd9534133be",
         "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
         "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts

@@ -41,6 +41,7 @@ export const EngineDescriptor = z.object({
     .regex(/[smdh]$/)
     .optional()
     .default('24h'),
+  timestampField: z.string().optional(),
   timeout: z
     .string()
     .regex(/[smdh]$/)

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml

@@ -36,6 +36,8 @@ components:
           type: string
           default: 24h
           pattern: '[smdh]$'
+        timestampField:
+          type: string
         timeout:
           type: string
           default: 180s

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts

@@ -28,6 +28,10 @@ export const InitEntityStoreRequestBody = z.object({
   filter: z.string().optional(),
   entityTypes: z.array(EntityType).optional(),
   enrichPolicyExecutionInterval: Interval.optional(),
+  /**
+   * The field to use as the timestamp.
+   */
+  timestampField: z.string().optional().default('@timestamp'),
   /**
    * The amount of time the transform looks back to calculate the aggregations.
    */

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml

@@ -33,6 +33,10 @@ paths:
                     $ref: './common.schema.yaml#/components/schemas/EntityType'
                 enrichPolicyExecutionInterval:
                   $ref: './common.schema.yaml#/components/schemas/Interval'
+                timestampField:
+                  type: string
+                  description: The field to use as the timestamp.
+                  default: '@timestamp'
                 lookbackPeriod: 
                   type: string
                   default: 24h

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.gen.ts

@@ -36,6 +36,10 @@ export const InitEntityEngineRequestBody = z.object({
   indexPattern: IndexPattern.optional(),
   filter: z.string().optional(),
   enrichPolicyExecutionInterval: Interval.optional(),
+  /**
+   * The field to use as the timestamp for the entity type.
+   */
+  timestampField: z.string().optional().default('@timestamp'),
   /**
    * The amount of time the transform looks back to calculate the aggregations.
    */

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml

@@ -35,7 +35,10 @@ paths:
                   type: string
                 enrichPolicyExecutionInterval:
                   $ref: '../common.schema.yaml#/components/schemas/Interval'
-
+                timestampField:
+                  type: string
+                  description: The field to use as the timestamp for the entity type.
+                  default: '@timestamp'
                 lookbackPeriod: 
                   type: string
                   default: 24h

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/start.gen.ts

@@ -21,7 +21,7 @@ import { EntityType } from '../common.gen';
 export type StartEntityEngineRequestParams = z.infer<typeof StartEntityEngineRequestParams>;
 export const StartEntityEngineRequestParams = z.object({
   /**
-   * The entity type of the engine (either 'user' or 'host').
+   * The entity type of the engine
    */
   entityType: EntityType,
 });

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/start.schema.yaml

@@ -16,7 +16,7 @@ paths:
           required: true
           schema:
             $ref: '../common.schema.yaml#/components/schemas/EntityType'
-          description: The entity type of the engine (either 'user' or 'host').
+          description: The entity type of the engine
       responses:
         '200':
           description: Successful response

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/entities/common.gen.ts

@@ -21,7 +21,7 @@ import { AssetCriticalityLevel } from '../../asset_criticality/common.gen';
 
 export type UserEntity = z.infer<typeof UserEntity>;
 export const UserEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -41,11 +41,16 @@ export const UserEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export type HostEntity = z.infer<typeof HostEntity>;
 export const HostEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -66,11 +71,16 @@ export const HostEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export type ServiceEntity = z.infer<typeof ServiceEntity>;
 export const ServiceEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -84,6 +94,11 @@ export const ServiceEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export const EntityInternal = z.union([UserEntity, HostEntity, ServiceEntity]);

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/entities/common.schema.yaml

@@ -9,7 +9,6 @@ components:
     UserEntity:
       type: object
       required:
-        - "@timestamp"
         - user
         - entity
       properties:
@@ -66,10 +65,15 @@ components:
               $ref: '../../asset_criticality/common.schema.yaml#/components/schemas/AssetCriticalityLevel'
           required:
             - criticality
+        event:
+          type: object
+          properties:
+            ingested:
+              type: string
+              format: date-time
     HostEntity:
       type: object
       required:
-        - "@timestamp"
         - host
         - entity
       properties:
@@ -130,10 +134,15 @@ components:
               $ref: '../../asset_criticality/common.schema.yaml#/components/schemas/AssetCriticalityLevel'
           required:
             - criticality
+        event:
+          type: object
+          properties:
+            ingested:
+              type: string
+              format: date-time
     ServiceEntity:
       type: object
       required:
-        - "@timestamp"
         - service
         - entity
       properties:
@@ -166,6 +175,12 @@ components:
               $ref: '../../asset_criticality/common.schema.yaml#/components/schemas/AssetCriticalityLevel'
           required:
             - criticality
+        event:
+          type: object
+          properties:
+            ingested:
+              type: string
+              format: date-time
     Entity:
       oneOf:
         - $ref: '#/components/schemas/UserEntity'

x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -346,6 +346,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp.
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -484,6 +488,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp for the entity type.
+                  type: string
         description: Schema for the engine initialization
         required: true
       responses:
@@ -500,7 +508,7 @@ paths:
     post:
       operationId: StartEntityEngine
       parameters:
-        - description: The entity type of the engine (either 'user' or 'host').
+        - description: The entity type of the engine
           in: path
           name: entityType
           required: true
@@ -1007,6 +1015,8 @@ components:
           default: 180s
           pattern: '[smdh]$'
           type: string
+        timestampField:
+          type: string
         type:
           $ref: '#/components/schemas/EntityType'
       required:
@@ -1146,6 +1156,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         host:
           type: object
           properties:
@@ -1184,7 +1200,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - host
         - entity
     IdField:
@@ -1293,6 +1308,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         service:
           type: object
           properties:
@@ -1303,7 +1324,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - service
         - entity
     StoreStatus:
@@ -1349,6 +1369,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         user:
           type: object
           properties:
@@ -1383,7 +1409,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - user
         - entity
   securitySchemes:

x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -346,6 +346,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp.
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -484,6 +488,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp for the entity type.
+                  type: string
         description: Schema for the engine initialization
         required: true
       responses:
@@ -500,7 +508,7 @@ paths:
     post:
       operationId: StartEntityEngine
       parameters:
-        - description: The entity type of the engine (either 'user' or 'host').
+        - description: The entity type of the engine
           in: path
           name: entityType
           required: true
@@ -1007,6 +1015,8 @@ components:
           default: 180s
           pattern: '[smdh]$'
           type: string
+        timestampField:
+          type: string
         type:
           $ref: '#/components/schemas/EntityType'
       required:
@@ -1146,6 +1156,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         host:
           type: object
           properties:
@@ -1184,7 +1200,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - host
         - entity
     IdField:
@@ -1293,6 +1308,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         service:
           type: object
           properties:
@@ -1303,7 +1324,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - service
         - entity
     StoreStatus:
@@ -1349,6 +1369,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         user:
           type: object
           properties:
@@ -1383,7 +1409,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - user
         - entity
   securitySchemes:

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts

@@ -21,6 +21,7 @@ export const DEFAULT_FREQUENCY = '1m';
 export const DEFAULT_DOCS_PER_SECOND = undefined;
 export const DEFAULT_INDEX_PATTERNS = '';
 export const DEFAULT_KQL_FILTER = '';
+export const DEFAULT_TIMESTAMP_FIELD = '@timestamp';
 
 export const defaultOptions: Expand<
   Required<Omit<InitEntityEngineRequestBody, 'docsPerSecond'>> & {
@@ -36,6 +37,7 @@ export const defaultOptions: Expand<
   indexPattern: DEFAULT_INDEX_PATTERNS,
   filter: DEFAULT_KQL_FILTER,
   enrichPolicyExecutionInterval: DEFAULT_INTERVAL,
+  timestampField: DEFAULT_TIMESTAMP_FIELD,
 };
 
 export const ENGINE_STATUS: Record<Uppercase<EngineStatus>, EngineStatus> = {

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts

@@ -24,4 +24,3 @@ export const DEFAULT_FIELD_HISTORY_LENGTH = 10;
 export const DEFAULT_SYNC_DELAY = '1m';
 export const DEFAULT_FREQUENCY = '1m';
 export const DEFAULT_LOOKBACK_PERIOD = '1d';
-export const DEFAULT_TIMESTAMP_FIELD = '@timestamp';

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts

@@ -6,25 +6,21 @@
  */
 
 import { assign, concat } from 'lodash/fp';
-
 import type {
   EntityType,
   InitEntityEngineRequestBody,
 } from '../../../../../common/api/entity_analytics';
-import { DEFAULT_TIMESTAMP_FIELD } from '../entity_definitions/constants';
 import { generateIndexMappings } from '../elasticsearch_assets';
 import {
   hostEntityEngineDescription,
   userEntityEngineDescription,
   universalEntityEngineDescription,
   serviceEntityEngineDescription,
 } from '../entity_definitions/entity_descriptions';
-
 import type { EntityStoreConfig } from '../types';
 import { buildEntityDefinitionId } from '../utils';
 import type { EntityDescription } from '../entity_definitions/types';
 import type { EntityEngineInstallationDescriptor } from './types';
-
 import { merge } from '../../../../../common/utils/objects/merge';
 import { defaultOptions } from '../constants';
 
@@ -64,7 +60,7 @@ export const createEngineDescription = (params: EngineDescriptionParams) => {
     frequency: options.frequency,
     docsPerSecond: options.docsPerSecond,
     lookbackPeriod: options.lookbackPeriod,
-    timestampField: description.settings?.timestampField || DEFAULT_TIMESTAMP_FIELD,
+    timestampField: options.timestampField,
   };
 
   const defaults = {

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor_type.ts

@@ -31,6 +31,9 @@ export const entityEngineDescriptorTypeMappings: SavedObjectsType['mappings'] =
       type: 'integer',
       index: false,
     },
+    timestampField: {
+      type: 'keyword', // timestampFieldName : @timestamp | event.ingested
+    },
   },
 };
 
@@ -40,6 +43,7 @@ const version1: SavedObjectsModelVersion = {
       type: 'mappings_addition',
       addedMappings: {
         fieldHistoryLength: { type: 'integer', index: false },
+        timestampField: { type: 'keyword' },
       },
     },
     {