From 65d9d9aea92fb77cf43f0905b78db362ba444cc6 Mon Sep 17 00:00:00 2001 From: Lucian Ioan <59661554+lucian-ioan@users.noreply.github.com> Date: Wed, 5 Mar 2025 05:28:56 +0200 Subject: [PATCH] [Azure] [PlatformLogs] Fix pipeline for edge cases (#12735) * fix pipeline * add changelog entry * bump manifest version * fix typo * convert eventduration to long * rerun tests * move logic to default pipeline * bump version * fix typo * change type: bugfix --- packages/azure/changelog.yml | 5 +++++ ...st-platformlogs-edgecases.log-expected.json | 2 +- ...platformlogs-identity-raw.log-expected.json | 2 +- ...-platformlogs-invalid-raw.log-expected.json | 2 +- .../test-platformlogs-kube.log-expected.json | 2 +- .../test-platformlogs-raw.log-expected.json | 2 +- ...t-platformlogs-remote-raw.log-expected.json | 2 +- .../ingest_pipeline/azure-shared-pipeline.yml | 18 ++++++------------ .../elasticsearch/ingest_pipeline/default.yml | 10 +++++++++- packages/azure/manifest.yml | 2 +- 10 files changed, 27 insertions(+), 20 deletions(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 5b22bd01d14..da95410c660 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.22.2" + changes: + - description: Fix Platform Logs pipeline inconsistent casing for subscription_id field. + type: bugfix + link: https://github.com/elastic/integrations/pull/12735 - version: 1.22.1 changes: - description: Fix the custom storage container description for the Azure Logs integration v2. diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json index d857e15ebfc..ce0ed6ee4e1 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-edgecases.log-expected.json @@ -117,4 +117,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json index ebe273ae56a..98351c4e433 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-identity-raw.log-expected.json @@ -46,4 +46,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json index fe2c5e46230..a751172707c 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json @@ -141,4 +141,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json index 992e568b905..0e8a4f27950 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-kube.log-expected.json @@ -41,4 +41,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json index 976ceeb00b4..6e35a099a2c 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json @@ -87,4 +87,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json index 7b1b1df1294..802f6bcfb7e 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json @@ -45,4 +45,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml index 7290fc00b5b..adfbaed90cd 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml @@ -7,8 +7,7 @@ processors: - grok: field: azure.resource_id patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule} - - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule} + - /(?i)subscriptions/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule} pattern_definitions: SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} GROUPID: .+ @@ -20,8 +19,7 @@ processors: field: azure.resource_id if: 'ctx.azure?.subscription_id == null' patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} - - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} + - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} pattern_definitions: SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} GROUPID: .+ @@ -32,8 +30,7 @@ processors: field: azure.resource_id if: 'ctx.azure?.subscription_id == null' patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} - - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} + - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} pattern_definitions: SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} GROUPID: .+ @@ -53,8 +50,7 @@ processors: field: azure.resource_id if: 'ctx.azure?.subscription_id == null' patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider} - - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider} + - /(?i)subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider} pattern_definitions: SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+ @@ -63,8 +59,7 @@ processors: field: azure.resource_id if: 'ctx.azure?.subscription_id == null' patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group} - - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group} + - /(?i)subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group} pattern_definitions: SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} GROUPID: .+ @@ -73,8 +68,7 @@ processors: field: azure.resource_id if: 'ctx.azure?.subscription_id == null' patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id} - - /subscriptions/%{SUBID:azure.subscription_id} + - /(?i)subscriptions/%{SUBID:azure.subscription_id} pattern_definitions: SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} ignore_failure: true diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml index 1e7b216435e..19a8e971524 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml @@ -91,6 +91,12 @@ processors: field: azure.platformlogs.resourceId target_field: azure.resource_id ignore_missing: true + if : ctx?.azure?.resource_id == null + - rename: + field: azure.platformlogs.ResourceId + target_field: azure.resource_id + ignore_missing: true + if : ctx?.azure?.resource_id == null - rename: field: azure.platformlogs.Region target_field: cloud.region @@ -148,8 +154,9 @@ processors: - convert: field: azure.platformlogs.durationMs target_field: event.duration - type: integer + type: long ignore_missing: true + ignore_failure: true if: "ctx?.azure?.platformlogs?.durationMs != null && ctx?.azure?.platformlogs?.durationMs instanceof String" - remove: field: azure.platformlogs.durationMs @@ -284,6 +291,7 @@ processors: - set: field: event.kind value: event + ignore_failure: true - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' - pipeline: diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 762a737f18a..df074461c79 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.22.1" +version: "1.22.2" description: This Elastic integration collects logs from Azure type: integration icons: