mimecast: implement siem_logs v2 agent (#11801)

Tested against a real endpoint. Pipeline test cases obtained from a test instance. Up to 10 examples of each available type are included. Not all types are represented.
elastic · Dec 9, 2024 · d19f5c2 · d19f5c2
1 parent beee275
commit d19f5c2
Show file tree

Hide file tree

Showing 21 changed files with 5,450 additions and 547 deletions.
diff --git a/packages/mimecast/_dev/deploy/docker/files/config.yml b/packages/mimecast/_dev/deploy/docker/files/config.yml
@@ -377,6 +377,139 @@ rules:
           }
           `}}
 
+  - path: /siem/v1/batch/events/cg
+    methods: ["GET"]
+    query_params:
+      type: "internal email protect"
+      nextPage: null
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [
+                  {
+                      "url": "http://svc-mimecast:8080/siemblob/iep",
+                      "expiry": "2024-11-19T02:14:04.839Z",
+                      "size": 424
+                  }
+              ],
+              "@nextPage": "nexttoken",
+              "isCaughtUp": false
+          }
+  - path: /siemblob/iep
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: '{{file "/files/iep.gz"}}'
+  - path: /siem/v1/batch/events/cg
+    methods: ["GET"]
+    query_params:
+      type: "internal email protect"
+      nextPage: "nexttoken"
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [],
+              "@nextPage": "String",
+              "isCaughtUp": true
+          }
+  - path: /siem/v1/batch/events/cg
+    methods: ["GET"]
+    query_params:
+      type: "receipt"
+      nextPage: null
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [
+                  {
+                      "url": "http://svc-mimecast:8080/siemblob/rec0",
+                      "expiry": "2024-11-19T02:14:04.839Z",
+                      "size": 511
+                  }
+              ],
+              "@nextPage": "nexttoken",
+              "isCaughtUp": false
+          }
+  - path: /siemblob/rec0
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: '{{file "/files/rec0.gz"}}'
+  - path: /siem/v1/batch/events/cg
+    methods: ["GET"]
+    query_params:
+      type: "receipt"
+      nextPage: "nexttoken"
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [
+                  {
+                      "url": "http://svc-mimecast:8080/siemblob/rec1",
+                      "expiry": "2024-11-19T02:14:04.839Z",
+                      "size": 572
+                  }
+              ],
+              "@nextPage": "lasttoken",
+              "isCaughtUp": false
+          }
+  - path: /siemblob/rec1
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: '{{file "/files/rec1.gz"}}'
+  - path: /siem/v1/batch/events/cg
+    methods: ["GET"]
+    query_params:
+      type: "receipt"
+      nextPage: "lasttoken"
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [],
+              "@nextPage": "String",
+              "isCaughtUp": true
+          }
+
   - path: /api/ttp/threat-intel/get-feed
     methods: ["POST"]
     request_body: /"feedType":"malware_customer","fileType":"stix","start":/

diff --git a/packages/mimecast/_dev/deploy/docker/files/iep.gz b/packages/mimecast/_dev/deploy/docker/files/iep.gz
diff --git a/packages/mimecast/_dev/deploy/docker/files/rec0.gz b/packages/mimecast/_dev/deploy/docker/files/rec0.gz
diff --git a/packages/mimecast/_dev/deploy/docker/files/rec1.gz b/packages/mimecast/_dev/deploy/docker/files/rec1.gz
diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.1.0"
+  changes:
+    - description: Add v2 API client for `siem_logs`.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11801
 - version: "2.0.0"
   changes:
     - description: Migrate message release logs data stream to Mimecast v2 API.

diff --git a/...s/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-docs-logs.log-expected.json b/...s/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-docs-logs.log-expected.json
@@ -23,6 +23,9 @@
             },
             "event": {
                 "action": "Acc",
+                "category": [
+                    "email"
+                ],
                 "created": "2017-05-26T16:47:41+0100",
                 "original": "{\"Act\":\"Acc\",\"Cphr\":\"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\",\"Dir\":\"Internal\",\"IP\":\"81.2.69.144\",\"MsgId\":\"<messageId@messageId>\",\"Rcpt\":\"[email protected]\",\"Sender\":\"[email protected]\",\"SpamInfo\":\"[]\",\"SpamLimit\":0,\"SpamProcessingDetail\":{\"spf\":{\"info\":\"SPF_FAIL\",\"allow\":true},\"dkim\":{\"info\":\"DKIM_UNKNOWN\",\"allow\":true}},\"SpamScore\":1,\"Subject\":\"message subject\",\"TlsVer\":\"TLSv1\",\"aCode\":\"7O7I7MvGP1mj8plHRDuHEA\",\"acc\":\"C0A0\",\"datetime\":\"2017-05-26T16:47:41+0100\",\"headerFrom\":\"[email protected]\"}",
                 "outcome": "unknown"
@@ -84,6 +87,9 @@
             },
             "event": {
                 "action": "Acc",
+                "category": [
+                    "email"
+                ],
                 "created": "2017-05-26T19:36:48+0100",
                 "original": "{\"Act\":\"Acc\",\"AttCnt\":2,\"AttNames\":\"\\\"filename.docx\\\", \\\"filename2.xlsx\\\"\",\"AttSize\":1267,\"MsgId\":\"messageId@mssageId\",\"MsgSize\":2116,\"aCode\":\"BY81J52RPjSmp7MrubnlZg\",\"acc\":\"C0A0\",\"datetime\":\"2017-05-26T19:36:48+0100\"}",
                 "outcome": "unknown"
@@ -114,6 +120,9 @@
             },
             "event": {
                 "action": "Acc",
+                "category": [
+                    "email"
+                ],
                 "created": "2017-05-26T19:36:48+0100",
                 "original": "{\"Act\":\"Acc\",\"AttCnt\":0,\"AttNames\":\"\",\"AttSize\":0,\"MsgId\":\"messageId@mssageId\",\"MsgSize\":2116,\"aCode\":\"BY81J52RPjSmp7MrubnlZg\",\"acc\":\"C0A0\",\"datetime\":\"2017-05-26T19:36:48+0100\"}",
                 "outcome": "unknown"
@@ -144,6 +153,9 @@
             },
             "event": {
                 "action": "Hld",
+                "category": [
+                    "email"
+                ],
                 "created": "2017-05-26T19:24:18+0100",
                 "original": "{\"Act\":\"Hld\",\"AttCnt\":0,\"AttNames\":\"\",\"AttSize\":0,\"Hld\":\"Spm\",\"IPInternalName\":\"false\",\"IPNewDomain\":\"false\",\"IPReplyMismatch\":\"false\",\"IPSimilarDomain\":\"false\",\"IPThreadDict\":\"false\",\"MsgId\":\"messageId@mssageId\",\"MsgSize\":56442,\"aCode\":\"015vTYvNN-Wn30v7M5MzNw\",\"acc\":\"C0A0\",\"datetime\":\"2017-05-26T19:24:18+0100\"}",
                 "outcome": "unknown",
@@ -191,6 +203,9 @@
                 }
             },
             "event": {
+                "category": [
+                    "email"
+                ],
                 "created": "2017-05-26T19:40:33+0100",
                 "original": "{\"AttCnt\":0,\"AttSize\":0,\"Attempt\":1,\"Cphr\":\"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\",\"Delivered\":true,\"Dir\":\"Inbound\",\"IP\":\"81.2.69.144\",\"Latency\":5618,\"MsgId\":\"messageId@mssageId\",\"Rcpt\":\"[email protected]\",\"ReceiptAck\":\"250 2.6.0 messageId@mssageId [InternalId=25473608] Queued mail for delivery\",\"Route\":\"Mimecast Exchange Rout\",\"Sender\":\"[email protected]\",\"Snt\":28237,\"Subject\":\"Auto Reply\",\"TlsVer\":\"TLSv1\",\"UseTls\":\"Yes\",\"aCode\":\"9q_HeIHHPYejZTBsnipWmQ\",\"acc\":\"C0A0\",\"datetime\":\"2017-05-26T19:40:33+0100\"}",
                 "outcome": "success"
@@ -262,6 +277,9 @@
                 }
             },
             "event": {
+                "category": [
+                    "email"
+                ],
                 "created": "2021-03-05T16:25:17+0000",
                 "original": "{\"CustomerIP\":\"true\",\"IP\":\"0.0.0.0\",\"MimecastIP\":\"false\",\"MsgId\":\"<[email protected]>\",\"Recipient\":\"[email protected]\",\"Route\":\"Inbound\",\"Sender\":\"[email protected]\",\"SenderDomain\":\"senderdomain.tld\",\"SenderDomainInternal\":\"false\",\"Size\":1648832,\"Subject\":\"Invoice Attached for payment\",\"Virus\":\"Anomali:Phishing\",\"acc\":\"C0A0\",\"datetime\":\"2021-03-05T16:25:17+0000\",\"fileExt\":\"xlsm\",\"fileMime\":\"application/vnd.ms-excel.sheet.macroEnabled.12\",\"fileName\":\"Invoice Attached for payment\",\"md5\":\"4dbe9dbfb53438d9ce410535355cd973\",\"sha1\":\"816b013c8be6e5708690645964b5d442c085041e\",\"sha256\":\"efe51c2453821310c7a34dca3054021d0f6d453b7133c381d75e3140901efd12\"}",
                 "outcome": "unknown"
@@ -306,6 +324,9 @@
                 }
             },
             "event": {
+                "category": [
+                    "email"
+                ],
                 "created": "2021-03-05T18:18:39+0000",
                 "original": "{\"MsgId\":\"<ABCDEF@domain-GHIK>\",\"Recipient\":\"[email protected]\",\"Route\":\"Inbound\",\"Sender\":\"[email protected]\",\"SenderDomain\":\"bdomain.tld\",\"SourceIP\":\"0.0.0.0\",\"Subject\":\"Opportunity to become VP\",\"aCode\":\"azYwczFKNga_v1sYBuJOvA\",\"acc\":\"C0A0\",\"datetime\":\"2021-03-05T18:18:39+0000\",\"headerFrom\":\"sender@adomain\"}",
                 "outcome": "unknown"
@@ -345,6 +366,9 @@
                 }
             },
             "event": {
+                "category": [
+                    "email"
+                ],
                 "created": "2021-03-04T21:31:08+0000",
                 "original": "{\"MsgId\":\"<CWXP123MB37349110AF6F6A2BC94F702EC4979@CWXP123MB3734.GBRP123.PROD.domain.tld>\",\"Recipient\":\"[email protected]\",\"Route\":\"Internal\",\"ScanResultInfo\":\"Blocked URL Category\",\"Sender\":\"[email protected]\",\"Subject\":\"Coffee Briefing\",\"URL\":\"https://domain.com/login/\",\"UrlCategory\":\"Phishing & Fraud\",\"aCode\":\"vit87EEXMPaEyl22Lrb92A\",\"acc\":\"C46A75\",\"datetime\":\"2021-03-04T21:31:08+0000\"}",
                 "outcome": "unknown"
@@ -386,6 +410,9 @@
             },
             "event": {
                 "action": "Hold",
+                "category": [
+                    "email"
+                ],
                 "created": "2020-07-27T00:39:59+0100",
                 "original": "{\"Action\":\"Hold\",\"CustomName\":\"false\",\"CustomThreatDictionary\":\"false\",\"Definition\":\"Default Impersonation Definition\",\"Hits\":\"1\",\"IP\":\"0.0.0.0\",\"InternalName\":\"true\",\"MsgId\":\"<[email protected]>\",\"NewDomain\":\"false\",\"Recipient\":\"recipient@domain\",\"ReplyMismatch\":\"false\",\"Route\":\"Inbound\",\"Sender\":\"sender@domain\",\"SimilarCustomExternalDomain\":\"false\",\"SimilarInternalDomain\":\"false\",\"SimilarMimecastExternalDomain\":\"false\",\"Subject\":\"Opportunity to become VP\",\"TaggedExternal\":\"false\",\"TaggedMalicious\":\"true\",\"ThreatDictionary\":\"false\",\"aCode\":\"q4qBpkoTOt-iStR7G44w3g\",\"acc\":\"C0A0\",\"datetime\":\"2020-07-27T00:39:59+0100\"}",
                 "outcome": "unknown"
@@ -437,6 +464,9 @@
                 }
             },
             "event": {
+                "category": [
+                    "email"
+                ],
                 "created": "2017-05-26T19:22:37+0100",
                 "original": "{\"acc\":\"C0A0\",\"datetime\":\"2017-05-26T19:22:37+0100\",\"reason\":\"malicious\",\"recipient\":\"[email protected]\",\"route\":\"inbound\",\"sender\":\"[email protected]\",\"senderDomain\":\"domain.com\",\"sourceIp\":\"81.2.69.144\",\"url\":\"http://bgmtechnology.com.au\",\"urlCategory\":\"Blocked\"}",
                 "outcome": "unknown",
@@ -501,6 +531,9 @@
                 }
             },
             "event": {
+                "category": [
+                    "email"
+                ],
                 "created": "2017-05-23T21:45:21+0100",
                 "original": "{\"IP\":\"81.2.69.144\",\"Recipient\":\"[email protected]\",\"Route\":\"Inbound\",\"Sender\":\"[email protected]\",\"SenderDomain\":\"domain.com\",\"Size\":378368,\"acc\":\"C1A1\",\"datetime\":\"2017-05-23T21:45:21+0100\",\"fileExt\":\"doc\",\"fileMime\":\"application/vnd.ms-office\",\"fileName\":\"1XCOLUMN.PVC\",\"md5\":\"7b52770644da336a9a59141c80807f37\",\"sha1\":\"a27850da9e7adfc8e1a94dabf2509fc9d65ee7e2\",\"sha256\":\"8746bb4b31ab6f03eb0a3b2c62ab7497658f0f85c8e7e82f042f9af0bb876d83\"}",
                 "outcome": "unknown"

diff --git a/...ogs/_dev/test/pipeline/test-siem-logs.log → .../_dev/test/pipeline/test-siem-v1-logs.log b/...ogs/_dev/test/pipeline/test-siem-logs.log → .../_dev/test/pipeline/test-siem-v1-logs.log