Panw cortex xdr enhancement test - DO NOT MERGE

.buildkite/hooks/pre-command

@@ -108,6 +108,11 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]
     if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
+
+        if [ -f "${GOOGLE_APPLICATION_CREDENTIALS:-""}" ]; then
+            GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)"
+            export GOOGLE_OAUTH_ACCESS_TOKEN
+        fi
     fi
 fi
 

.buildkite/scripts/common.sh

@@ -807,7 +807,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort | grep -E '^(panw_cortex_xdr)$'
 }
 
 check_package() {

packages/panw_cortex_xdr/_dev/build/docs/README.md

@@ -1,28 +1,120 @@
 # Palo Alto Cortex XDR Integration
 
-The PANW XDR integration collects alerts with multiple events from the [Cortex XDR Alerts API](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR-REST-API/Get-all-Alerts) and incidents from [Cortex XDR Incidents API](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR-REST-API/IncidentManagement).
+[Palo Alto Networks Cortex XDR](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR/Cortex-XDR-Pro-Administrator-Guide/Overview) is a powerful security solution that gives you complete visibility into network traffic, user behavior, and endpoint activity. It streamlines threat investigation by correlating data from various sensors, uncovering attack timelines and root causes with clarity. With Cortex XDR, you can take immediate response actions and minimize threat impact. Additionally, it enables proactive threat prevention through customizable detection rules, including BIOCs, IOCs, and Correlation Rules, helping you stay ahead of emerging threats.
 
-## Logs
+The Palo Alto Cortex XDR integration collects [alerts](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR-REST-API/Get-Alerts-Multi-Events-v1) and [incidents](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR-REST-API/Get-all-Incidents) via the REST API and supports [endpoint event forwarding](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR/Cortex-XDR-Pro-Administrator-Guide/Manage-Event-Forwarding) through Google Cloud Storage (GCS).
+
+## Data streams
+
+**[Alerts](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR-REST-API/Get-Alerts-Multi-Events-v1)** – Retrieves alerts generated by Cortex XDR based on raw endpoint data. A single alert may include one or more local endpoint events, each of which is ingested as a separate document in Elasticsearch. Requires both an API Key and API Key ID.
+
+**[Incidents](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR-REST-API/Get-all-Incidents)** – Retrieves incidents generated by Cortex XDR. Each incident may include one or more endpoint events, each stored as a separate document in Elasticsearch. When an incident is modified in the Cortex XDR UI (e.g., severity or status changes, alerts linked), a new document reflecting the updated values is indexed. Requires both an API Key and API Key ID.
+
+**[Event](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR/Cortex-XDR-Pro-Administrator-Guide/Manage-Event-Forwarding)** - Retrieves events forwarded by Cortex XDR to Google Cloud Storage (GCS).
+
+## Requirements
+
+### Agentless enabled integration
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments.  This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
+### Agent-based installation
+
+Elastic Agent must be installed. For more details, check the Elastic Agent [installation instructions](docs-content://reference/fleet/install-elastic-agents.md).
+
+## Compatibility
+
+This integration is compatible with Palo Alto Cortex XDR version **V3.13**. For the REST API, it has been tested with versions **v1** and **v2**.
+
+## Setup
+
+### To collect logs through REST API, follow the below steps:
+
+**Note:** To collect logs from API version v1, use the **HTTP JSON** input type. To collect logs from API version v2, use the **CEL** input type.
+
+1. Log in to your Cortex XDR portal.
+2. Navigate to **Settings > Configurations > Integrations > API Keys**.
+3. Select **+ New Key** to generate a new API key.
+4. Choose the type of API key you want to generate: **Standard** or **Advanced**, based on the desired security level.
+5. (Optional) To define a time limit for the API key, enable **Expiration Date** and set the desired expiration.
+6. Select the **access level** for this key:
+   - Choose from existing **Roles**, or
+   - Select **Custom** for granular permission settings.
+7. Click **Generate** to create the key.
+8. **Copy the API Key** shown in the notification and store it securely — you won’t be able to view it again.
+9. Locate the **API Key ID** in the **ID** column of the API Keys table. This represents your `x-xdr-auth-id`.
+10. To get your **Domain Name**, right-click the API key and select **View Examples**.
+    - Copy the domain part from the **CURL example URL** (e.g., `https://api-{fqdn}/public_api/...`).
+
+**Reference:** [Cortex XDR – Get Started with APIs](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR-REST-API/Get-Started-with-Cortex-XDR-APIs)
+
+### To collect endpoint logs using Event Forwarding, follow the below steps:
+
+1. In your Cortex XDR portal, navigate to:
+   **Settings > Configurations > Data Management > Event Forwarding**.
+2. In the **Activation** section, enable **Endpoints Event Forwarding** to export raw endpoint data for Cortex XDR Pro EP and Cloud Endpoints.
+   - A temporary GCS bucket is automatically created by Cortex XDR(Cortex XDR uses GCS as the storage destination by default).
+   - Logs are stored as line-delimited JSON gzip files.
+3. In the **Destination** section:
+   - Copy the GCS storage path.
+   - Download the Service Account JSON Web Token (JWT) file containing access credentials.
+4. To configure data collection in Elastic, use the following details:
+   - **Project ID**
+   - **Service Account Key** (downloaded JSON file)
+   - **Bucket path** (copied from the GCS destination)
+
+**Reference:** [Cortex XDR - Manage Event Forwarding](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR/Cortex-XDR-Pro-Administrator-Guide/Manage-Event-Forwarding)
+
+**Note:** The endpoint event forwarding feature requires a Cortex XDR Pro license along with the Event Forwarding add-on. Enabling this feature requires Administrator access.
+
+### Enabling the integration in Elastic
+
+1. In Kibana, navigate to Management > Integrations.
+2. In the "Search for integrations" bar, search for Palo Alto Cortex XDR.
+3. Select the "Palo Alto Cortex XDR" integration from the search results.
+4. Click "Add Palo Alto Cortex XDR" to add the integration.
+5. Configure the integration with the required parameters based on your preferred method of data collection:
+  - For REST API (applies to both Alerts and Incidents with HTTP JSON and CEL input):
+    - API Domain
+    - API Token
+    - API Token ID
+    - Interval
+    - Initial Interval
+  - For GCS (used for event collection via endpoint event forwarding):
+    - Project ID
+    - Service Account Key or Service Account Credentials File
+    - Buckets
+6. Click **Save and continue** to complete the integration setup.
+
+## Logs reference
 
 ### Alerts
 
-The Cortex XDR Alerts API is used to retrieve alerts generated by Cortex XDR based on raw endpoint data. A single alert might include one or more local endpoint events, each event generating its own document on Elasticsearch.
+This is the `Alerts` dataset.
+
+#### Example
 
-The Palo Alto XDR integration requires both an API key and API key ID, both which can be retrieved from the Cortex XDR UI. See: [Get Started with Cortex XDR API](https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-api/cortex-xdr-api-overview/get-started-with-cortex-xdr-apis.html)
+{{fields "alerts"}}
 
 {{event "alerts"}}
 
-{{fields "alerts"}}
+### Event
 
-### Incidents
+This is the `Event` dataset.
 
-The Cortex XDR Incidents API is used to retrieve incidents generated by Cortex XDR based on raw endpoint data. A single incident might include one or more local endpoint events, each event generating its own document on Elasticsearch.
+#### Example
 
-The Palo Alto XDR integration requires both an API key and API key ID, both which can be retrieved from the Cortex XDR UI. See: [Get Started with Cortex XDR API](https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-api/cortex-xdr-api-overview/get-started-with-cortex-xdr-apis.html)
+{{fields "event"}}
 
-When a Cortex XDR Incident is modified in the Cortex XDR UI (e.g. severity or status changed, additional alerts linked) it will be indexed as a new document with the new values.
+{{event "event"}}
 
-{{event "incidents"}}
+### Incidents
+
+This is the `Incidents` dataset.
+
+#### Example
 
 {{fields "incidents"}}
 
+{{event "incidents"}}

packages/panw_cortex_xdr/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.2.0"
+  changes:
+    - description: Add event data stream to collect event forwarding logs via the Google Cloud Storage input and add dashboards for alert and incident data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13680
 - version: "2.1.2"
   changes:
     - description: Updated description to ssl nodes to be consistent with other packages descriptions.

packages/panw_cortex_xdr/_dev/deploy/docker/http-mock-config.yml → packages/panw_cortex_xdr/data_stream/alerts/_dev/deploy/docker/http-mock-config.yml

@@ -727,85 +727,3 @@ rules:
                 ]
              }
           }
-  - path: /public_api/v1/incidents/get_incidents
-    methods:
-      - POST
-    request_headers:
-      Authorization: testing-token
-      X-Xdr-Auth-Id: 1
-    request_body: /.*/
-    responses:
-      - status_code: 200
-        headers:
-          Content-Type:
-            - application/json
-        body: |-
-          {
-            "reply": {
-            "total_count": 390,
-            "result_count": 1,
-            "incidents": [
-                {
-                  "incident_id": "893",
-                  "incident_name": null,
-                  "creation_time": 1691976000230,
-                  "modification_time": 1691976000230,
-                  "detection_time": null,
-                  "status": "new",
-                  "severity": "low",
-                  "description": "'Large Upload (Generic)' generated by XDR Analytics detected on host test1234 involving user nt authority\\system",
-                  "assigned_user_mail": null,
-                  "assigned_user_pretty_name": null,
-                  "alert_count": 1,
-                  "low_severity_alert_count": 1,
-                  "med_severity_alert_count": 0,
-                  "high_severity_alert_count": 0,
-                  "critical_severity_alert_count": 0,
-                  "user_count": 1,
-                  "host_count": 1,
-                  "notes": null,
-                  "resolve_comment": null,
-                  "resolved_timestamp": null,
-                  "manual_severity": null,
-                  "manual_description": null,
-                  "xdr_url": "https://test.xdr.eu.paloaltonetworks.com/incident-view?caseId=893",
-                  "starred": false,
-                  "hosts": [
-                      "test1234:b567c1a651e66999158aef5d864dad25"
-                  ],
-                  "users": [
-                      "nt authority\\system"
-                  ],
-                  "incident_sources": [
-                      "XDR Analytics"
-                  ],
-                  "rule_based_score": null,
-                  "predicted_score": 5,
-                  "manual_score": null,
-                  "aggregated_score": 5,
-                  "wildfire_hits": 0,
-                  "alerts_grouping_status": "Enabled",
-                  "mitre_tactics_ids_and_names": [
-                      "TA0010 - Exfiltration"
-                  ],
-                  "mitre_techniques_ids_and_names": [
-                      "T1048 - Exfiltration Over Alternative Protocol"
-                  ],
-                  "alert_categories": [
-                      "Exfiltration"
-                  ],
-                  "original_tags": [
-                      "DS:PANW/XDR Agent",
-                      "EG:win-server-ex-ransomeware_report",
-                      "EG:win-server-default"
-                  ],
-                  "tags": [
-                      "DS:PANW/XDR Agent",
-                      "EG:win-server-default",
-                      "EG:win-server-ex-ransomeware_report"
-                  ]
-                }
-            ],
-            "restricted_incident_ids": []
-            }
-          }

packages/panw_cortex_xdr/data_stream/alerts/_dev/test/pipeline/test-panw-xdr-bioc.log-expected.json

@@ -1247,4 +1247,4 @@
             }
         }
     ]
-}
+}

packages/panw_cortex_xdr/data_stream/alerts/_dev/test/pipeline/test-panw-xdr.log-expected.json

@@ -849,4 +849,4 @@
         },
         null
     ]
-}
+}

packages/panw_cortex_xdr/data_stream/alerts/_dev/test/pipeline/test-reprocess.json-expected.json

@@ -122,4 +122,4 @@
             ]
         }
     ]
-}
+}

packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml

@@ -14,6 +14,12 @@ processors:
       target_field: event.original
       ignore_missing: true
       if: ctx.event?.original == null
+  - remove:
+      field: message
+      tag: remove_message
+      ignore_missing: true
+      description: The `message` field is no longer required if the document has an `event.original` field.
+      if: ctx.event?.original != null
   - set:
       field: event.kind
       value: alert