mimecast: set event.kind:alert for appropriate events (#12835)

elastic · Mar 3, 2025 · 4df3a48 · 4df3a48
1 parent 0978854
commit 4df3a48
Show file tree

Hide file tree

Showing 25 changed files with 290 additions and 17 deletions.
diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.6.0"
+  changes:
+    - description: Set `event.kind:"alert"` for relevant events.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12835
 - version: "2.5.2"
   changes:
     - description: Add missing ECS field mappings.

diff --git a/...ecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log b/...ecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log
@@ -18,3 +18,4 @@
 {"_offset":1815702,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xxg1c71KpzFB8T-j9zj9y3f91byjes4ysqetgk19g_1732525893","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"821aff47dea6b57c6cb6bd262738eeabd28e2659f2ac0cb3ee490828d3a143f4_1732525893","subtype":null,"timestamp":1732525893398,"type":"mailflow"}
 {"_offset":1820908,"_partition":53,"accountId":"AUS2474","aggregateId":"4XyPl23Kn4z84ly-hnj5fgu9nbwnghx86mj18qp44b_1732630590","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"3125eb6ff78c055eb7449a47286a453dcb66e176d2c751a6236bba4232c6fe31_1732630590","subtype":null,"timestamp":1732630590705,"type":"mailflow"}
 {"_offset":1803841,"_partition":53,"accountId":"AUS2474","aggregateId":"4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206","subtype":null,"timestamp":1732212206960,"type":"mailflow"}
+{"attachments":["tpsreport.xlsx"],"subject":"RE: Your archive mailbox is almost full.","senderEnvelope":"[email protected]","messageId":"messageId","threatState":"DELIVERED","senderHeader":"[email protected]","source":"OFFICE_365_MAIL","type":"entities","tags":["UNTRUSTWORTHY","SPAM"],"accountId":"C0A0","aggregateId":"aggregateId","processingId":"processingId","threatType":"POLICIES_DISABLED","recipients":["[email protected]"],"policiesApplied":[{"action":null,"mode":null,"name":"Default O365 Mail policy"},{"action":null,"mode":null,"name":"Default O365 Mail policy"}],"historicalMail":false,"subtype":"POLICIES_DISABLED","senderIp":"81.2.69.144","timestamp":1689685037899,"direction":"Inbound"}
diff --git a/...eam/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json b/...eam/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json
@@ -348,6 +348,7 @@
                     "email"
                 ],
                 "created": "2024-11-18T15:24:35.250Z",
+                "kind": "alert",
                 "original": "{\"_offset\":1790506,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XsWdG63WDzFy33-kckoq8rx19ibc7iccjsjx6hsxd_1731943475\",\"attachments\":[\"Sandbox Test.xlsx\"],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<[email protected]>\",\"policiesApplied\":[{\"action\":\"BLOCK\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"66e03b099e150698fc62f354796f2baa94c2f625a34ac92a0c7e8eb4a2afb11c_1731943475\",\"recipients\":[\"[email protected]\"],\"senderEnvelope\":\"[email protected]\",\"senderHeader\":\"\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Message from Node-RED\",\"subtype\":\"MALWARE\",\"tags\":[\"MALWARE\"],\"threatState\":\"BLOCKED\",\"threatType\":\"MALWARE\",\"timestamp\":1731943475250,\"type\":\"entities\"}"
             },
             "mimecast": {
@@ -1204,6 +1205,95 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2023-07-18T12:57:17.899Z",
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "email": {
+                "attachments": [
+                    {
+                        "file": {
+                            "name": [
+                                "tpsreport.xlsx"
+                            ]
+                        }
+                    }
+                ],
+                "direction": "inbound",
+                "from": {
+                    "address": [
+                        "[email protected]"
+                    ]
+                },
+                "message_id": "messageId",
+                "subject": "RE: Your archive mailbox is almost full.",
+                "to": {
+                    "address": [
+                        "[email protected]"
+                    ]
+                }
+            },
+            "event": {
+                "category": [
+                    "email"
+                ],
+                "created": "2023-07-18T12:57:17.899Z",
+                "kind": "alert",
+                "original": "{\"attachments\":[\"tpsreport.xlsx\"],\"subject\":\"RE: Your archive mailbox is almost full.\",\"senderEnvelope\":\"[email protected]\",\"messageId\":\"messageId\",\"threatState\":\"DELIVERED\",\"senderHeader\":\"[email protected]\",\"source\":\"OFFICE_365_MAIL\",\"type\":\"entities\",\"tags\":[\"UNTRUSTWORTHY\",\"SPAM\"],\"accountId\":\"C0A0\",\"aggregateId\":\"aggregateId\",\"processingId\":\"processingId\",\"threatType\":\"POLICIES_DISABLED\",\"recipients\":[\"[email protected]\"],\"policiesApplied\":[{\"action\":null,\"mode\":null,\"name\":\"Default O365 Mail policy\"},{\"action\":null,\"mode\":null,\"name\":\"Default O365 Mail policy\"}],\"historicalMail\":false,\"subtype\":\"POLICIES_DISABLED\",\"senderIp\":\"81.2.69.144\",\"timestamp\":1689685037899,\"direction\":\"Inbound\"}"
+            },
+            "mimecast": {
+                "accountId": "C0A0",
+                "aggregateId": "aggregateId",
+                "historicalMail": false,
+                "log_type": "entities",
+                "policiesApplied": [
+                    {
+                        "name": "Default O365 Mail policy"
+                    },
+                    {
+                        "name": "Default O365 Mail policy"
+                    }
+                ],
+                "processingId": "processingId",
+                "senderHeader": "[email protected]",
+                "source": "OFFICE_365_MAIL",
+                "subtype": "POLICIES_DISABLED",
+                "tags": [
+                    "UNTRUSTWORTHY",
+                    "SPAM"
+                ],
+                "threatState": "DELIVERED",
+                "threatType": "POLICIES_DISABLED"
+            },
+            "related": {
+                "ip": [
+                    "81.2.69.144"
+                ],
+                "user": [
+                    "[email protected]",
+                    "[email protected]"
+                ]
+            },
+            "source": {
+                "geo": {
+                    "city_name": "London",
+                    "continent_name": "Europe",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lat": 51.5142,
+                        "lon": -0.0931
+                    },
+                    "region_iso_code": "GB-ENG",
+                    "region_name": "England"
+                },
+                "ip": "81.2.69.144"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
-}
+}
diff --git a/...ages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml b/...ages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml
@@ -34,6 +34,10 @@ processors:
       field: event.created
       copy_from: '@timestamp'
       if: ctx['@timestamp'] != null
+  - set:
+      field: event.kind
+      value: alert
+      if: ctx.mimecast?.tags instanceof List && ctx.mimecast.tags.length != 0
 
   ### NOTE LOG TYPE
   - rename:

diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-common-config.yml
@@ -1,3 +1,6 @@
 fields:
+  _conf:
+    alerting:
+      - block
   tags:
     - preserve_original_event
diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log
@@ -13,4 +13,5 @@
 {"senderAddress":"[email protected]","recipientAddress":"[email protected]","subject":"Re","eventTime":"2024-11-17T20:57:30+0000","route":"outbound","policy":"Confidential","action":"hold","messageId":"<[email protected]>"}
 {"senderAddress":"[email protected]","recipientAddress":"[email protected]","subject":"FIRE DRILL","eventTime":"2024-11-17T20:16:02+0000","route":"inbound","policy":"Confidential","action":"hold","messageId":"<[email protected]>"}
 {"senderAddress":"[email protected]","recipientAddress":"[email protected]","subject":"New CERA.com Coming Soon! - CERA Alert","eventTime":"2024-11-17T19:47:39+0000","route":"inbound","policy":"Confidential","action":"hold","messageId":"<[email protected]>"}
+{"senderAddress":"[email protected]","recipientAddress":"[email protected]","subject":"New CERA.com Coming Soon! - CERA Alert","eventTime":"2024-11-17T19:47:39+0000","route":"inbound","policy":"Confidential","action":"block","messageId":"<[email protected]>"}
 {"meta":{"status":200,"pagination":{"pageSize":10,"totalCount":519,"next":"nextToken"}},"data":[],"fail":[]}
diff --git a/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json b/packages/mimecast/data_stream/dlp_logs/_dev/test/pipeline/test-dlp-logs.log-expected.json
@@ -525,6 +525,42 @@
                 "preserve_original_event"
             ]
         },
+        {
+            "@timestamp": "2024-11-17T19:47:39.000Z",
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "email": {
+                "direction": "inbound",
+                "from": {
+                    "address": [
+                        "[email protected]"
+                    ]
+                },
+                "message_id": "<[email protected]>",
+                "subject": "New CERA.com Coming Soon! - CERA Alert",
+                "to": {
+                    "address": [
+                        "[email protected]"
+                    ]
+                }
+            },
+            "event": {
+                "action": "block",
+                "category": [
+                    "email"
+                ],
+                "created": "2024-11-17T19:47:39+0000",
+                "kind": "alert",
+                "original": "{\"senderAddress\":\"[email protected]\",\"recipientAddress\":\"[email protected]\",\"subject\":\"New CERA.com Coming Soon! - CERA Alert\",\"eventTime\":\"2024-11-17T19:47:39+0000\",\"route\":\"inbound\",\"policy\":\"Confidential\",\"action\":\"block\",\"messageId\":\"<[email protected]>\"}"
+            },
+            "rule": {
+                "name": "Confidential"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
         null
     ]
-}
+}
diff --git a/packages/mimecast/data_stream/dlp_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/dlp_logs/agent/stream/cel.yml.hbs
@@ -155,6 +155,13 @@ program: |
       )
     )
   )
+{{#if alerting}}
+_conf:
+  alerting:
+{{#each alerting as |a|}}
+    - {{a}}
+{{/each}}
+{{/if}}
 tags:
 {{#if preserve_original_event}}
   - preserve_original_event

diff --git a/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs b/packages/mimecast/data_stream/dlp_logs/agent/stream/httpjson.yml.hbs
@@ -42,6 +42,13 @@ response.pagination:
 cursor:
   next_date:
     value: '[[.first_event.eventTime]]'
+{{#if alerting}}
+_conf:
+  alerting:
+{{#each alerting as |a|}}
+    - {{a}}
+{{/each}}
+{{/if}}
 tags:
 {{#if preserve_original_event}}
 - preserve_original_event

diff --git a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml
@@ -26,6 +26,10 @@ processors:
   - set:
       field: event.category
       value: [email]
+  - set:
+      field: event.kind
+      value: alert
+      if: ctx._conf?.alerting instanceof List && ctx._conf.alerting.contains(ctx.mimecast?.action)
 
   - date:
       description: Use 'mimecast.eventTime' as the '@timestamp'
@@ -90,6 +94,7 @@ processors:
       description: Cleanup of repeated/unwanted/temporary fields.
       field: 
         - mimecast
+        - _conf
       ignore_missing: true
       
 # Error handling

diff --git a/packages/mimecast/data_stream/dlp_logs/manifest.yml b/packages/mimecast/data_stream/dlp_logs/manifest.yml
@@ -14,6 +14,14 @@ streams:
         required: true
         show_user: false
         default: 5m
+      - name: alerting
+        type: text
+        title: Alert Actions
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - block
       - name: tags
         type: text
         title: Tags
@@ -78,6 +86,15 @@ streams:
         multi: false
         required: true
         show_user: false
+      - name: alerting
+        type: text
+        title: Alert Actions
+        description: The set of DLP actions that should be classified as an alert. Possible values are delete, hold, bouce, smart_folder, disable_smart_folder, content_expire, meta_expire, stationery, disable_stationery, gcc, secure_delivery, delivery_route, document_policy, disable_document_policy, secure_messaging, disable_secure_messaging_policy, attach_set_policy, remove_email, tag, link, block, none, and notification.
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - block
       - name: tags
         type: text
         title: Tags