[Azure] [PlatformLogs] Fix pipeline for edge cases (#12735)

* fix pipeline * add changelog entry * bump manifest version * fix typo * convert eventduration to long * rerun tests * move logic to default pipeline * bump version * fix typo * change type: bugfix
elastic · Mar 5, 2025 · 65d9d9a · 65d9d9a
1 parent d682f74
commit 65d9d9a
Show file tree

Hide file tree

Showing 10 changed files with 27 additions and 20 deletions.
diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.22.2"
+  changes:
+    - description: Fix Platform Logs pipeline inconsistent casing for subscription_id field.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/12735
 - version: 1.22.1
   changes:
     - description: Fix the custom storage container description for the Azure Logs integration v2.

diff --git a/...data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json b/...data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json
@@ -117,4 +117,4 @@
             ]
         }
     ]
-}
+}
diff --git a/...a_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json b/...a_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json
@@ -46,4 +46,4 @@
             ]
         }
     ]
-}
+}
diff --git a/...ta_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json b/...ta_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json
@@ -141,4 +141,4 @@
             ]
         }
     ]
-}
+}
diff --git a/...zure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json b/...zure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json
@@ -41,4 +41,4 @@
             ]
         }
     ]
-}
+}
diff --git a/...azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json b/...azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json
@@ -87,4 +87,4 @@
             ]
         }
     ]
-}
+}
diff --git a/...ata_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json b/...ata_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json
@@ -45,4 +45,4 @@
             ]
         }
     ]
-}
+}
diff --git a/...es/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/...es/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -7,8 +7,7 @@ processors:
   - grok:
       field: azure.resource_id
       patterns:
-      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}
-      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule}
+      - /(?i)subscriptions/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}
       pattern_definitions:
         SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
         GROUPID: .+
@@ -20,8 +19,7 @@ processors:
       field: azure.resource_id
       if: 'ctx.azure?.subscription_id == null'
       patterns:
-      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
-      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+      - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
       pattern_definitions:
         SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
         GROUPID: .+
@@ -32,8 +30,7 @@ processors:
       field: azure.resource_id
       if: 'ctx.azure?.subscription_id == null'
       patterns:
-        - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
-        - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+        - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
       pattern_definitions:
         SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
         GROUPID: .+
@@ -53,8 +50,7 @@ processors:
       field: azure.resource_id
       if: 'ctx.azure?.subscription_id == null'
       patterns:
-        - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}
-        - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider}
+        - /(?i)subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider}
       pattern_definitions:
         SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
         PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
@@ -63,8 +59,7 @@ processors:
       field: azure.resource_id
       if: 'ctx.azure?.subscription_id == null'
       patterns:
-        - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}
-        - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}
+        - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}
       pattern_definitions:
         SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
         GROUPID: .+
@@ -73,8 +68,7 @@ processors:
       field: azure.resource_id
       if: 'ctx.azure?.subscription_id == null'
       patterns:
-        - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}
-        - /subscriptions/%{SUBID:azure.subscription_id}
+        - /(?i)subscriptions/%{SUBID:azure.subscription_id}
       pattern_definitions:
         SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
       ignore_failure: true

diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml
@@ -91,6 +91,12 @@ processors:
       field: azure.platformlogs.resourceId
       target_field: azure.resource_id
       ignore_missing: true
+      if : ctx?.azure?.resource_id == null
+  - rename:
+      field: azure.platformlogs.ResourceId
+      target_field: azure.resource_id
+      ignore_missing: true
+      if : ctx?.azure?.resource_id == null
   - rename:
       field: azure.platformlogs.Region
       target_field: cloud.region
@@ -148,8 +154,9 @@ processors:
   - convert:
       field: azure.platformlogs.durationMs
       target_field: event.duration
-      type: integer
+      type: long
       ignore_missing: true
+      ignore_failure: true
       if: "ctx?.azure?.platformlogs?.durationMs != null && ctx?.azure?.platformlogs?.durationMs instanceof String"
   - remove:
       field: azure.platformlogs.durationMs
@@ -284,6 +291,7 @@ processors:
   - set:
       field: event.kind
       value: event
+      ignore_failure: true
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
   - pipeline:

diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml
@@ -1,6 +1,6 @@
 name: azure
 title: Azure Logs
-version: "1.22.1"
+version: "1.22.2"
 description: This Elastic integration collects logs from Azure
 type: integration
 icons:
-Original file line number
+Diff line change
@@ Expand Up / @@ -117,4 +117,4 @@ @@
                 ]
             }
         ]
-    }
+    }