[azure,o365,microsoft_defender_endpoint,m365_defender] Standardize ECS in Microsoft Integrations

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.24.0"
+  changes:
+    - description: Set `service.id`, `device.id`, `user.id`, `session.id`, and `token.id` in graphactivitylogs dataset.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13931
 - version: "1.23.3"
   changes:
     - description: Improve Azure logs documentation with more details on log categories routing rules.

packages/azure/data_stream/graphactivitylogs/_dev/test/pipeline/test-activitylogs-raw.log

@@ -2,4 +2,5 @@
 {"Level":4,"callerIpAddress":"81.2.69.143","category":"MicrosoftGraphActivityLogs","correlationId":"f7739jk0-e6d1-4e3f-985a-64937fbge367","durationMs":453011,"location":"Germany West Central","operationName":"Microsoft Graph Activity","operationVersion":"v1.0","properties":{"apiVersion":"v1.0","appId":"a5a68e12-268a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"2","clientRequestId":"2fe56790-a848-4c83-9d2c-5675972aejk9","durationMs":453011,"identityProvider":"https://sts.windows.net/aa30985b-427d-4434-b4dc-8f9040719adb/","ipAddress":"81.2.69.143","location":"Germany West Central","operationId":"f7739jk0-e6d1-4e3f-985a-64937fbge367","requestId":"f7739jk0-e6d1-4e3f-985a-64937fbge367","requestMethod":"GET","requestUri":"https://graph.microsoft.com/v1.0/me/photos/96x96/$value","responseSizeBytes":294,"responseStatusCode":404,"roles":null,"scopes":"email openid Organization.Read.All Policy.ReadWrite.ApplicationConfiguration profile User.Read","servicePrincipalId":null,"signInActivityId":"sign-in_ActivityId","tenantId":"aa30985b-427d-4434-b4dc-8f9040719adb","timeGenerated":"2024-03-07T10:35:31.9597832Z","tokenIssuedAt":"2024-03-07T10:30:30Z","userAgent":"","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":"1997b4d3-0g8d-90cb-bhj5-d80n3122e98 1997b4d3-0f8d-76cb-bhj5-d80n3122e98"},"resourceId":"/TENANTS/AA30985B-427D-4434-B4DC-8F9040719ADB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"404","tenantId":"aa30985b-427d-4434-b4dc-8f9040719adb","time":"2024-03-07T10:35:31.9597832Z"}
 {"Level":4,"callerIpAddress":"81.2.69.144","category":"MicrosoftGraphActivityLogs","correlationId":"f7739da0-e6d1-4e3f-875a-64934fbge347","durationMs":16688471,"location":"UK South","operationName":"Microsoft Graph Activity","operationVersion":"beta","properties":{"apiVersion":"beta","appId":"a5a68e12-268a-3c91-a5f2-b9254e67hb28","atContent":"","clientAuthMethod":"0","clientRequestId":"2fe56789-a848-4c93-9s2c-5675972aghk9","durationMs":16688471,"identityProvider":null,"ipAddress":"81.2.69.144","location":"UK South","operationId":"f7739da0-e6d1-4e3f-875a-64934fbge347","requestId":"f7739da0-e6d1-4e3f-875a-64934fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/beta//users/7ef3c2ad-d52l-4a89-8cf9-c30178181027/photos/48x48/$value","responseSizeBytes":0,"responseStatusCode":404,"roles":null,"scopes":"AdministrativeUnit.ReadWrite.All AuditLog.Read.All Directory.AccessAsUser.All Directory.Write.Restricted email openid Organization.Read.All Policy.ReadWrite.Authorization profile User.EnableDisableAccount.All User.ReadWrite.All","servicePrincipalId":null,"signInActivityId":"signin_ActivityId","tenantId":"ab47545b-420e-46fg-c4dc-8f7697k1aadb","timeGenerated":"2024-03-07T16:42:22.84914Z","tokenIssuedAt":"2024-03-07T16:37:20Z","userAgent":"","userId":"285e0849-a706-4a9a-9eb1-f4e21cc78793","wids":"1997b4d3-0g8d-90cb-bhj5-d80n3122e98 1997b4d3-0g8d-90cb-bhj5-d80n3122e99 1997b4d3-0g8d-90cb-bhj5-d80n3122e80 1997b4d3-0g8d-90cb-bhj5-d80n3122e83"},"resourceId":"/TENANTS/AB47545B-420E-46FG-C4DC-8F7697K1AADB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"404","tenantId":"ab47545b-420e-46fg-c4dc-8f7697k1aadb","time":"2024-03-07T16:42:22.8491400Z"}
 {"Level":4,"callerIpAddress":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","category":"MicrosoftGraphActivityLogs","correlationId":"f7749da0-e6g1-4f3f-975a-64937fbge347","durationMs":846544,"location":"UK South","operationName":"Microsoft Graph Activity","operationVersion":"beta","properties":{"apiVersion":"beta","appId":"a5a68e32-269a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"0","clientRequestId":"2fe56789-a848-4c93-9d2d-5675972ardk9","durationMs":846544,"identityProvider":null,"ipAddress":"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6","location":"UK South","operationId":"f7749da0-e6g1-4f3f-975a-64937fbge347","requestId":"f7749da0-e6g1-4f3f-975a-64937fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/beta/devices?$select=displayName,id,deviceId\\u0026$search=%22displayName:a%22\\u0026$top=30","responseSizeBytes":0,"responseStatusCode":200,"roles":null,"scopes":"AccessReview.ReadWrite.All AuditLog.Read.All ChangeManagement.Read.All ConsentRequest.Create ConsentRequest.Read ConsentRequest.ReadApprove.All ConsentRequest.ReadWrite.All CustomSecAttributeAuditLogs.Read.All Directory.AccessAsUser.All Directory.Read.All Directory.ReadWrite.All Directory.Write.Restricted DirectoryRecommendations.Read.All DirectoryRecommendations.ReadWrite.All email EntitlementManagement.Read.All Group.ReadWrite.All IdentityProvider.ReadWrite.All IdentityRiskEvent.ReadWrite.All IdentityRiskyServicePrincipal.ReadWrite.All IdentityRiskyUser.ReadWrite.All IdentityUserFlow.Read.All LifecycleWorkflows.ReadWrite.All openid Policy.Read.All Policy.Read.IdentityProtection Policy.ReadWrite.AuthenticationFlows Policy.ReadWrite.AuthenticationMethod Policy.ReadWrite.ConditionalAccess Policy.ReadWrite.ExternalIdentities Policy.ReadWrite.IdentityProtection Policy.ReadWrite.MobilityManagement profile Reports.Read.All RoleManagement.ReadWrite.Directory RoleManagement.ReadWrite.Exchange SecurityEvents.ReadWrite.All TrustFrameworkKeySet.Read.All User.Export.All User.ReadWrite.All UserAuthenticationMethod.ReadWrite.All","servicePrincipalId":null,"signInActivityId":"signin_Activity-Id","tenantId":"ab47545b-420e-46fg-c4dc-8f7697k1aadb","timeGenerated":"2024-03-07T16:42:12.0485843Z","tokenIssuedAt":"2024-03-07T16:36:51Z","userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0","userId":"285e0849-a706-4a9a-9eb1-f4e21cc78793","wids":"a207b4d3-0g8d-90cb-bhj5-d80n3122e67 a207b4d3-0g8d-90cb-bhj5-d80n3122e69 a207b4d3-0g8d-90cb-bhj5-d80n3122e89"},"resourceId":"/TENANTS/AB47545B-420E-46FG-C4DC-8F7697K1AADB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab47545b-420e-46fg-c4dc-8f7697k1aadb","time":"2024-03-07T16:42:12.0485843Z"}
-{"Level":4,"callerIpAddress":"81.2.69.143","category":"MicrosoftGraphActivityLogs","correlationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","durationMs":1100725,"location":"France Central","operationName":"Microsoft Graph Activity","operationVersion":"v1.0","properties":{"apiVersion":"v1.0","appId":"a5a68e32-269a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"2","clientRequestId":"2fe58790-a848-4a93-9d2c-5645972aejk9","durationMs":1100725,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","location":"France Central","operationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/v1.0/directoryRoles","responseSizeBytes":4300,"responseStatusCode":200,"roles":"Application.Read.All Domain.Read.All GroupMember.Read.All LicenseAssignment.ReadWrite.All Organization.Read.All Policy.Read.ConditionalAccess RoleManagement.Read.Directory Team.ReadBasic.All TeamsTab.Create TeamsTab.Read.All TeamsTab.ReadWrite.All User.Read.All","scopes":null,"servicePrincipalId":"f2aq4c71-31e3-5065-91g3-4b2dfbsv50fg","signInActivityId":"sign-in_ActivityId","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2024-03-07T10:24:44.7939418Z","tokenIssuedAt":"2024-03-07T10:19:44Z","userAgent":"","userId":null,"wids":"a207b4d3-0g8d-90cb-bhj5-d80n3121e69"},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2024-03-07T10:24:44.7939418Z"}
+{"Level":4,"callerIpAddress":"81.2.69.143","category":"MicrosoftGraphActivityLogs","correlationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","durationMs":1100725,"location":"France Central","operationName":"Microsoft Graph Activity","operationVersion":"v1.0","properties":{"apiVersion":"v1.0","appId":"a5a68e32-269a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"2","clientRequestId":"2fe58790-a848-4a93-9d2c-5645972aejk9","durationMs":1100725,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","location":"France Central","operationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/v1.0/directoryRoles","responseSizeBytes":4300,"responseStatusCode":200,"roles":"Application.Read.All Domain.Read.All GroupMember.Read.All LicenseAssignment.ReadWrite.All Organization.Read.All Policy.Read.ConditionalAccess RoleManagement.Read.Directory Team.ReadBasic.All TeamsTab.Create TeamsTab.Read.All TeamsTab.ReadWrite.All User.Read.All","scopes":null,"servicePrincipalId":"f2aq4c71-31e3-5065-91g3-4b2dfbsv50fg","signInActivityId":"sign-in_ActivityId","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2024-03-07T10:24:44.7939418Z","tokenIssuedAt":"2024-03-07T10:19:44Z","userAgent":"","userId":null,"wids":"a207b4d3-0g8d-90cb-bhj5-d80n3121e69"},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2024-03-07T10:24:44.7939418Z"}
+{"Level":4,"callerIpAddress":"81.2.69.143","category":"MicrosoftGraphActivityLogs","correlationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","durationMs":1100725,"location":"France Central","operationName":"Microsoft Graph Activity","operationVersion":"v1.0","properties":{"apiVersion":"v1.0","appId":"a5a68e32-269a-3c91-a5e2-b9254e67hb29","atContent":"","clientAuthMethod":"2","C_DeviceId": "abc123","c_Sid":"xyz000","clientRequestId":"2fe58790-a848-4a93-9d2c-5645972aejk9","durationMs":1100725,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","location":"France Central","operationId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestId":"f7839da0-e7d1-4e4f-985a-64937fbge347","requestMethod":"GET","requestUri":"https://graph.microsoft.com/v1.0/directoryRoles","responseSizeBytes":4300,"responseStatusCode":200,"roles":"Application.Read.All Domain.Read.All GroupMember.Read.All LicenseAssignment.ReadWrite.All Organization.Read.All Policy.Read.ConditionalAccess RoleManagement.Read.Directory Team.ReadBasic.All TeamsTab.Create TeamsTab.Read.All TeamsTab.ReadWrite.All User.Read.All","scopes":null,"servicePrincipalId":"f2aq4c71-31e3-5065-91g3-4b2dfbsv50fg","signInActivityId":"sign-in_ActivityId","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2024-03-07T10:24:44.7939418Z","tokenIssuedAt":"2024-03-07T10:19:44Z","userAgent":"","userId":null,"wids":"a207b4d3-0g8d-90cb-bhj5-d80n3121e69"},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2024-03-07T10:24:44.7939418Z"}