atlassian_{bitbucket,confluence,jira}: improve pipeline robustness to…

… empty strings Atlassian will send empty string values rather than omit fields, so null out all empty string values at the start of processing. Also improve error reporting and script robustness.
elastic · Mar 6, 2025 · 4cf6aeb · 4cf6aeb
1 parent 65d9d9a
commit 4cf6aeb
Show file tree

Hide file tree

Showing 28 changed files with 403 additions and 38 deletions.
diff --git a/packages/atlassian_bitbucket/changelog.yml b/packages/atlassian_bitbucket/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "2.5.0"
+  changes:
+    - description: Improve error reporting.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12977
+    - description: Handle empty string values more robustly.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/12977
 - version: "2.4.0"
   changes:
     - description: Update Kibana constraint to support 9.0.0.

diff --git a/...ian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api-empty.log-expected.json b/...ian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api-empty.log-expected.json
@@ -2,4 +2,4 @@
     "expected": [
         null
     ]
-}
+}
diff --git a/packages/atlassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api.log b/packages/atlassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api.log
@@ -174,4 +174,5 @@
 {"timestamp":"2021-11-27T17:29:00.687Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.apps","category":"Apps","actionI18nKey":"bitbucket.service.plugin.audit.action.pluginenabled","action":"Plugin enabled"},"affectedObjects":[{"name":"com.atlassian.templaterenderer.api","type":"MISC"}],"changedValues":[],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.service.plugin.audit.attribute.version","name":"Version","value":"4.1.4"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"com.atlassian.templaterenderer.api"}]}
 {"timestamp":"2021-11-27T17:26:26.205Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.globaladministration","category":"Global administration","actionI18nKey":"bitbucket.service.applicationconfiguration.audit.action.displaynamechanged","action":"Server name changed"},"affectedObjects":[],"changedValues":[{"key":"Name","i18nKey":"bitbucket.service.applicationconfiguration.audit.changedvalue.displaynamechanged.name","to":"Bitbucket"}],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.audit.attribute.legacy.details","name":"details","value":"{\"new\":\"Bitbucket\",\"old\":null}"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"DISPLAY_NAME"}]}
 {"timestamp":"2021-11-27T17:26:25.141Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.usersandgroups","category":"Users and groups","actionI18nKey":"bitbucket.service.user.audit.action.groupcreated","action":"User group created"},"affectedObjects":[{"name":"stash-users","type":"GROUP","uri":"http://bitbucket.internal:7990/admin/groups/view?name=stash-users","id":"stash-users"}],"changedValues":[],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"stash-users"}]}
-{"timestamp":"2021-11-27T17:26:25.045Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.usersandgroups","category":"Users and groups","actionI18nKey":"bitbucket.service.user.audit.action.directorycreated","action":"User directory created"},"affectedObjects":[],"changedValues":[],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.service.user.audit.attribute.directory.name","name":"Directory name","value":"Bitbucket Internal Directory"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"Bitbucket Internal Directory"}]}
+{"timestamp":"2021-11-27T17:26:25.045Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.usersandgroups","category":"Users and groups","actionI18nKey":"bitbucket.service.user.audit.action.directorycreated","action":"User directory created"},"affectedObjects":[],"changedValues":[],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.service.user.audit.attribute.directory.name","name":"Directory name","value":"Bitbucket Internal Directory"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"Bitbucket Internal Directory"}]}
+{"timestamp":"2021-11-27T17:38:53.360Z","author":{"name":"admin","type":"NORMAL","id":"2","uri":"http://bitbucket.internal:7990/users/admin","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.usersandgroups","category":"Users and groups","actionI18nKey":"bitbucket.service.user.audit.action.userrenamed","action":"Username changed"},"affectedObjects":[{"name":"test.user","type":"USER","id":"3"}],"changedValues":[{"key":"Username","i18nKey":"bitbucket.service.user.audit.attribute.user.name","from":"test","to":"test.user"}],"source":"","system":"http://bitbucket.internal:7990","node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"Browser","extraAttributes":[{"nameI18nKey":"bitbucket.audit.attribute.legacy.details","name":"details","value":"{\"oldUsername\":\"test\"}"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"test.user"}]}
diff --git a/...atlassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api.log-expected.json b/...atlassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api.log-expected.json
@@ -10243,6 +10243,89 @@
                 "id": "-1",
                 "name": "System"
             }
+        },
+        {
+            "@timestamp": "2021-11-27T17:38:53.360Z",
+            "bitbucket": {
+                "audit": {
+                    "affected_objects": [
+                        {
+                            "id": "3",
+                            "name": "test.user",
+                            "type": "USER"
+                        }
+                    ],
+                    "changed_values": [
+                        {
+                            "from": "test",
+                            "i18nKey": "bitbucket.service.user.audit.attribute.user.name",
+                            "key": "Username",
+                            "to": "test.user"
+                        }
+                    ],
+                    "extra_attributes": [
+                        {
+                            "name": "details",
+                            "nameI18nKey": "bitbucket.audit.attribute.legacy.details",
+                            "value": "{\"oldUsername\":\"test\"}"
+                        },
+                        {
+                            "name": "target",
+                            "nameI18nKey": "bitbucket.audit.attribute.legacy.target",
+                            "value": "test.user"
+                        }
+                    ],
+                    "method": "Browser",
+                    "type": {
+                        "action": "Username changed",
+                        "actionI18nKey": "bitbucket.service.user.audit.action.userrenamed",
+                        "category": "Users and groups",
+                        "categoryI18nKey": "bitbucket.service.audit.category.usersandgroups"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "bitbucket.service.user.audit.action.userrenamed",
+                "category": [
+                    "iam"
+                ],
+                "kind": "event",
+                "original": "{\"timestamp\":\"2021-11-27T17:38:53.360Z\",\"author\":{\"name\":\"admin\",\"type\":\"NORMAL\",\"id\":\"2\",\"uri\":\"http://bitbucket.internal:7990/users/admin\",\"avatarUri\":\"\"},\"type\":{\"categoryI18nKey\":\"bitbucket.service.audit.category.usersandgroups\",\"category\":\"Users and groups\",\"actionI18nKey\":\"bitbucket.service.user.audit.action.userrenamed\",\"action\":\"Username changed\"},\"affectedObjects\":[{\"name\":\"test.user\",\"type\":\"USER\",\"id\":\"3\"}],\"changedValues\":[{\"key\":\"Username\",\"i18nKey\":\"bitbucket.service.user.audit.attribute.user.name\",\"from\":\"test\",\"to\":\"test.user\"}],\"source\":\"\",\"system\":\"http://bitbucket.internal:7990\",\"node\":\"8767044c-1b98-4d64-82db-ef29af8c3792\",\"method\":\"Browser\",\"extraAttributes\":[{\"nameI18nKey\":\"bitbucket.audit.attribute.legacy.details\",\"name\":\"details\",\"value\":\"{\\\"oldUsername\\\":\\\"test\\\"}\"},{\"nameI18nKey\":\"bitbucket.audit.attribute.legacy.target\",\"name\":\"target\",\"value\":\"test.user\"}]}",
+                "type": [
+                    "user",
+                    "change"
+                ]
+            },
+            "related": {
+                "hosts": [
+                    "bitbucket.internal"
+                ],
+                "user": [
+                    "admin",
+                    "test",
+                    "test.user"
+                ]
+            },
+            "service": {
+                "address": "http://bitbucket.internal:7990"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "changes": {
+                    "name": "test.user"
+                },
+                "id": "2",
+                "name": "admin",
+                "target": {
+                    "id": "3",
+                    "name": "test"
+                }
+            }
         }
     ]
-}
+}
diff --git a/...lassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-files.log-expected.json b/...lassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-files.log-expected.json
@@ -7865,4 +7865,4 @@
             }
         }
     ]
-}
+}
diff --git a/packages/atlassian_bitbucket/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/atlassian_bitbucket/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -14,6 +14,32 @@ processors:
       target_field: json
   - drop:
       if: ctx.json?.entities instanceof List && ctx.json.entities.length == 0
+  - script:
+      lang: painless
+      if: ctx.json instanceof Map
+      description: This script processor iterates over the whole document to remove fields with null values.
+      source: |
+        void handleMap(Map map) {
+          for (def x : map.values()) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+          }
+          map.values().removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
+        }
+        void handleList(List list) {
+          for (def x : list) {
+              if (x instanceof Map) {
+                  handleMap(x);
+              } else if (x instanceof List) {
+                  handleList(x);
+              }
+          }
+          list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
+        }
+        handleMap(ctx.json);
   - set:
       field: _tmp.timestamp
       copy_from: json.timestamp

diff --git a/packages/atlassian_bitbucket/manifest.yml b/packages/atlassian_bitbucket/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: atlassian_bitbucket
 title: Atlassian Bitbucket
-version: "2.4.0"
+version: "2.5.0"
 description: Collect logs from Atlassian Bitbucket with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/atlassian_confluence/changelog.yml b/packages/atlassian_confluence/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.29.0"
+  changes:
+    - description: Improve error reporting.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12977
+    - description: Handle empty string values more robustly.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/12977
 - version: "1.28.0"
   changes:
     - description: Update Kibana constraint to support 9.0.0.

diff --git a/...an_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api-empty.log-expected.json b/...an_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api-empty.log-expected.json
@@ -2,4 +2,4 @@
     "expected": [
         null
     ]
-}
+}
diff --git a/packages/atlassian_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api.log b/packages/atlassian_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api.log
@@ -179,4 +179,5 @@
 {"timestamp":"2021-11-23T00:34:44.466Z","author":{"name":"Anonymous","type":"user","id":"-2","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.plugins","category":"Apps","actionI18nKey":"audit.logging.summary.plugin.enabled","action":"App enabled"},"affectedObjects":[{"name":"Confluence Collaborative Editor Plugin","type":"App"}],"changedValues":[],"source":"81.2.69.143","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
 {"timestamp":"2021-11-28T17:05:37.142Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.renamed","action":"User renamed"},"affectedObjects":[{"name":"asdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[{"key":"Username","i18nKey":"audit.logging.changed.value.username","from":"asdf","to":"asdf123"}],"source":"10.100.100.2","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
 {"timestamp":"2021-11-28T17:06:11.805Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.updated","action":"User details updated"},"affectedObjects":[{"name":"asdf asdfasdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[],"source":"10.100.100.2","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
-{"timestamp":"2021-11-28T17:05:37.158Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.updated","action":"User details updated"},"affectedObjects":[{"name":"asdf asdfasdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[{"key":"Display name","i18nKey":"Display name","from":"asdf","to":"asdf asdfasdf"}],"source":"10.100.100.2","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
+{"timestamp":"2021-11-28T17:05:37.158Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.updated","action":"User details updated"},"affectedObjects":[{"name":"asdf asdfasdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[{"key":"Display name","i18nKey":"Display name","from":"asdf","to":"asdf asdfasdf"}],"source":"10.100.100.2","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
+{"timestamp":"2021-11-28T17:05:37.158Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.updated","action":"User details updated"},"affectedObjects":[{"name":"asdf asdfasdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[{"key":"Display name","i18nKey":"Display name","from":"asdf","to":"asdf asdfasdf"}],"source":"","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
diff --git a/...tlassian_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api.log-expected.json b/...tlassian_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api.log-expected.json
@@ -16410,6 +16410,79 @@
                     "name": "asdf123"
                 }
             }
+        },
+        {
+            "@timestamp": "2021-11-28T17:05:37.158Z",
+            "confluence": {
+                "audit": {
+                    "affected_objects": [
+                        {
+                            "id": "2c9680837d4a3682017d67821e520003",
+                            "name": "asdf asdfasdf",
+                            "type": "User",
+                            "uri": "http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123"
+                        }
+                    ],
+                    "changed_values": [
+                        {
+                            "from": "asdf",
+                            "i18nKey": "Display name",
+                            "key": "Display name",
+                            "to": "asdf asdfasdf"
+                        }
+                    ],
+                    "method": "Browser",
+                    "type": {
+                        "action": "User details updated",
+                        "actionI18nKey": "audit.logging.summary.user.updated",
+                        "category": "Users and groups",
+                        "categoryI18nKey": "audit.logging.category.user.management"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "audit.logging.summary.user.updated",
+                "category": [
+                    "iam"
+                ],
+                "kind": "event",
+                "original": "{\"timestamp\":\"2021-11-28T17:05:37.158Z\",\"author\":{\"name\":\"Joe Bob\",\"type\":\"user\",\"id\":\"2c9680837d4a3682017d4a375a280000\",\"uri\":\"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123\",\"avatarUri\":\"\"},\"type\":{\"categoryI18nKey\":\"audit.logging.category.user.management\",\"category\":\"Users and groups\",\"actionI18nKey\":\"audit.logging.summary.user.updated\",\"action\":\"User details updated\"},\"affectedObjects\":[{\"name\":\"asdf asdfasdf\",\"type\":\"User\",\"uri\":\"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123\",\"id\":\"2c9680837d4a3682017d67821e520003\"}],\"changedValues\":[{\"key\":\"Display name\",\"i18nKey\":\"Display name\",\"from\":\"asdf\",\"to\":\"asdf asdfasdf\"}],\"source\":\"\",\"system\":\"http://confluence.internal:8090\",\"method\":\"Browser\",\"extraAttributes\":[]}",
+                "type": [
+                    "user",
+                    "change"
+                ]
+            },
+            "related": {
+                "hosts": [
+                    "confluence.internal"
+                ],
+                "user": [
+                    "admin123",
+                    "asdf123"
+                ]
+            },
+            "service": {
+                "address": "http://confluence.internal:8090"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "changes": {
+                    "full_name": "asdf asdfasdf"
+                },
+                "full_name": "Joe Bob",
+                "id": "2c9680837d4a3682017d4a375a280000",
+                "name": "admin123",
+                "target": {
+                    "full_name": "asdf",
+                    "id": "2c9680837d4a3682017d67821e520003",
+                    "name": "asdf123"
+                }
+            }
         }
     ]
-}
+}
diff --git a/..._confluence/data_stream/audit/_dev/test/pipeline/test-audit-cloud-empty.log-expected.json b/..._confluence/data_stream/audit/_dev/test/pipeline/test-audit-cloud-empty.log-expected.json
@@ -2,4 +2,4 @@
     "expected": [
         null
     ]
-}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,4 +2,4 @@ @@
         "expected": [
             null
         ]
-    }
+    }