85 log analytics private ingestion works

HoussemDellai · Feb 28, 2024 · 038d6cf · 038d6cf
1 parent acd1daa
commit 038d6cf
Show file tree

Hide file tree

Showing 11 changed files with 418 additions and 367 deletions.
diff --git a/85_prometheus_grafana_private_endpoint/ampls.tf b/85_prometheus_grafana_private_endpoint/ampls.tf
@@ -18,17 +18,17 @@ resource "azurerm_monitor_private_link_scoped_service" "ampls-dce-log-analytics"
 }
 
 
+# # not required
+# # resource "azurerm_monitor_private_link_scoped_service" "prometheus" {
+# #   name                = "ampls-prometheus"
+# #   resource_group_name = azurerm_resource_group.rg_monitoring.name
+# #   scope_name          = azurerm_monitor_private_link_scope.ampls.name
+# #   linked_resource_id  = azurerm_monitor_workspace.prometheus.id
+# # }
 
-# resource "azurerm_monitor_private_link_scoped_service" "prometheus" {
-#   name                = "ampls-prometheus"
+# resource "azurerm_monitor_private_link_scoped_service" "dce-prometheus" {
+#   name                = "ampls-dce-prometheus"
 #   resource_group_name = azurerm_resource_group.rg_monitoring.name
 #   scope_name          = azurerm_monitor_private_link_scope.ampls.name
-#   linked_resource_id  = azurerm_monitor_workspace.prometheus.id
-# }
-
-resource "azurerm_monitor_private_link_scoped_service" "dce-prometheus" {
-  name                = "ampls-dce-prometheus"
-  resource_group_name = azurerm_resource_group.rg_monitoring.name
-  scope_name          = azurerm_monitor_private_link_scope.ampls.name
-  linked_resource_id  = azurerm_monitor_data_collection_endpoint.dce-prometheus.id
-}
+#   linked_resource_id  = azurerm_monitor_data_collection_endpoint.dce-prometheus.id
+# }
diff --git a/85_prometheus_grafana_private_endpoint/data_collection_rule.tf b/85_prometheus_grafana_private_endpoint/data_collection_rule.tf
diff --git a/...vate_endpoint/data_collection_endpoint.tf → ...ana_private_endpoint/dce-log_analytics.tf b/...vate_endpoint/data_collection_endpoint.tf → ...ana_private_endpoint/dce-log_analytics.tf
@@ -1,14 +1,13 @@
-resource "azurerm_monitor_data_collection_endpoint" "dce-prometheus" {
-  name                          = "dce-prometheus"
+resource "azurerm_monitor_data_collection_endpoint" "dce-log-analytics" {
+  name                          = "dce-log-analytics"
   resource_group_name           = azurerm_resource_group.rg_monitoring.name
   location                      = azurerm_resource_group.rg_monitoring.location
-  kind                          = "Linux"
-  public_network_access_enabled = false # true # false
+  public_network_access_enabled = false
 }
 
 # associate to a Data Collection Endpoint
-resource "azurerm_monitor_data_collection_rule_association" "dce-aks-prometheus" {
+resource "azurerm_monitor_data_collection_rule_association" "dcra-dce-log-analytics-aks" {
   name                        = "configurationAccessEndpoint" # name is required when data_collection_rule_id is specified. And when data_collection_endpoint_id is specified, the name is populated with configurationAccessEndpoint
   target_resource_id          = azurerm_kubernetes_cluster.aks.id
-  data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.dce-prometheus.id
-}
+  data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.dce-log-analytics.id
+}
diff --git a/85_prometheus_grafana_private_endpoint/dce-prometheus.tf b/85_prometheus_grafana_private_endpoint/dce-prometheus.tf
@@ -0,0 +1,14 @@
+# resource "azurerm_monitor_data_collection_endpoint" "dce-prometheus" {
+#   name                          = "dce-prometheus"
+#   resource_group_name           = azurerm_resource_group.rg_monitoring.name
+#   location                      = azurerm_resource_group.rg_monitoring.location
+#   kind                          = "Linux"
+#   public_network_access_enabled = false # true # false
+# }
+
+# # associate to a Data Collection Endpoint
+# resource "azurerm_monitor_data_collection_rule_association" "dcra-dce-prometheus-aks" {
+#   name                        = "configurationAccessEndpoint" # "dcra-dce-prometheus-aks" #  # name is required when data_collection_rule_id is specified. And when data_collection_endpoint_id is specified, the name is populated with configurationAccessEndpoint
+#   target_resource_id          = azurerm_kubernetes_cluster.aks.id
+#   data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.dce-prometheus.id
+# }
diff --git a/...private_endpoint/dce-dcr-log_analytics.tf → ...ana_private_endpoint/dcr-log_analytics.tf b/...private_endpoint/dce-dcr-log_analytics.tf → ...ana_private_endpoint/dcr-log_analytics.tf
@@ -1,17 +1,3 @@
-resource "azurerm_monitor_data_collection_endpoint" "dce-log-analytics" {
-  name                          = "dce-log-analytics"
-  resource_group_name           = azurerm_resource_group.rg_monitoring.name
-  location                      = azurerm_resource_group.rg_monitoring.location
-  public_network_access_enabled = false
-}
-
-# associate to a Data Collection Endpoint
-resource "azurerm_monitor_data_collection_rule_association" "dce-aks-log-analytics" {
-  name                        = "configurationAccessEndpoint" # name is required when data_collection_rule_id is specified. And when data_collection_endpoint_id is specified, the name is populated with configurationAccessEndpoint
-  target_resource_id          = azurerm_kubernetes_cluster.aks.id
-  data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.dce-log-analytics.id
-}
-
 resource "azurerm_monitor_data_collection_rule" "dcr-log-analytics" {
   name                        = "dcr-log-analytics"
   resource_group_name         = azurerm_resource_group.rg_monitoring.name
@@ -66,8 +52,8 @@ resource "azurerm_monitor_data_collection_rule" "dcr-log-analytics" {
 }
 
 # associate to a Data Collection Rule
-resource "azurerm_monitor_data_collection_rule_association" "dcr-aks-log-analytics" {
-  name                    = "dcr-aks-log-analytics"
+resource "azurerm_monitor_data_collection_rule_association" "dcra-dcr-log-analytics-aks" {
+  name                    = "dcra-dcr-log-analytics-aks"
   target_resource_id      = azurerm_kubernetes_cluster.aks.id
   data_collection_rule_id = azurerm_monitor_data_collection_rule.dcr-log-analytics.id
 }
diff --git a/85_prometheus_grafana_private_endpoint/dcr-prometheus.tf b/85_prometheus_grafana_private_endpoint/dcr-prometheus.tf
@@ -0,0 +1,35 @@
+# resource "azurerm_monitor_data_collection_rule" "dcr-prometheus" {
+#   name                        = "dcr-prometheus"
+#   resource_group_name         = azurerm_resource_group.rg_monitoring.name
+#   location                    = azurerm_resource_group.rg_monitoring.location
+#   data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.dce-prometheus.id
+#   kind                        = "Linux"
+#   description                 = "DCR for Azure Monitor Metrics Profile (Managed Prometheus)"
+
+#   data_sources {
+#     prometheus_forwarder {
+#       name    = "PrometheusDataSource"
+#       streams = ["Microsoft-PrometheusMetrics"]
+#     }
+#   }
+
+#   destinations {
+#     monitor_account {
+#       monitor_account_id = azurerm_monitor_workspace.prometheus.id
+#       name               = azurerm_monitor_workspace.prometheus.name
+#     }
+#   }
+
+#   data_flow {
+#     streams      = ["Microsoft-PrometheusMetrics"]
+#     destinations = [azurerm_monitor_workspace.prometheus.name]
+#   }
+# }
+
+# # associate to a Data Collection Rule
+# resource "azurerm_monitor_data_collection_rule_association" "dcra-dcr-prometheus-aks" {
+#   name                    = "dcra-dcr-prometheus-aks"
+#   target_resource_id      = azurerm_kubernetes_cluster.aks.id
+#   data_collection_rule_id = azurerm_monitor_data_collection_rule.dcr-prometheus.id
+#   description             = "Association of data collection rule. Deleting this association will break the data collection for this AKS Cluster."
+# }
diff --git a/85_prometheus_grafana_private_endpoint/grafana.tf b/85_prometheus_grafana_private_endpoint/grafana.tf
@@ -1,48 +1,48 @@
-resource "azurerm_dashboard_grafana" "grafana" {
-  name                              = var.grafana_name
-  resource_group_name               = azurerm_resource_group.rg_monitoring.name
-  location                          = azurerm_resource_group.rg_monitoring.location
-  api_key_enabled                   = true
-  deterministic_outbound_ip_enabled = true
-  public_network_access_enabled     = true
-  sku                               = "Standard"
-  zone_redundancy_enabled           = false
-  grafana_major_version             = "10" # 9
-
-  azure_monitor_workspace_integrations {
-    resource_id = azurerm_monitor_workspace.prometheus.id
-  }
-
-  identity {
-    type         = "SystemAssigned" # "UserAssigned" # 
-    # identity_ids = [azurerm_user_assigned_identity.identity-grafana.id]
-  }
-}
-
-data "azurerm_client_config" "current" {}
-
-resource "azurerm_role_assignment" "role_grafana_admin" {
-  scope                = azurerm_dashboard_grafana.grafana.id
-  role_definition_name = "Grafana Admin"
-  principal_id         = data.azurerm_client_config.current.object_id
-}
-
-resource "azurerm_role_assignment" "role_monitoring_data_reader" {
-  scope                = azurerm_monitor_workspace.prometheus.id
-  role_definition_name = "Monitoring Data Reader"
-  principal_id         = azurerm_dashboard_grafana.grafana.identity.0.principal_id # azurerm_user_assigned_identity.identity-grafana.principal_id # 
-}
-
-data "azurerm_subscription" "current" {}
-
-resource "azurerm_role_assignment" "role_monitoring_reader" {
-  scope                = data.azurerm_subscription.current.id
-  role_definition_name = "Monitoring Reader"
-  principal_id         = azurerm_dashboard_grafana.grafana.identity.0.principal_id # azurerm_user_assigned_identity.identity-grafana.principal_id # 
-}
-
-# resource "azurerm_user_assigned_identity" "identity-grafana" {
-#   name                = "identity-grafana"
-#   resource_group_name = azurerm_resource_group.rg_monitoring.name
-#   location            = azurerm_resource_group.rg_monitoring.location
+# resource "azurerm_dashboard_grafana" "grafana" {
+#   name                              = var.grafana_name
+#   resource_group_name               = azurerm_resource_group.rg_monitoring.name
+#   location                          = azurerm_resource_group.rg_monitoring.location
+#   api_key_enabled                   = true
+#   deterministic_outbound_ip_enabled = true
+#   public_network_access_enabled     = true
+#   sku                               = "Standard"
+#   zone_redundancy_enabled           = false
+#   grafana_major_version             = "10" # 9
+
+#   azure_monitor_workspace_integrations {
+#     resource_id = azurerm_monitor_workspace.prometheus.id
+#   }
+
+#   identity {
+#     type = "SystemAssigned" # "UserAssigned" # 
+#     # identity_ids = [azurerm_user_assigned_identity.identity-grafana.id]
+#   }
 # }
+
+# data "azurerm_client_config" "current" {}
+
+# resource "azurerm_role_assignment" "role_grafana_admin" {
+#   scope                = azurerm_dashboard_grafana.grafana.id
+#   role_definition_name = "Grafana Admin"
+#   principal_id         = data.azurerm_client_config.current.object_id
+# }
+
+# resource "azurerm_role_assignment" "role_monitoring_data_reader" {
+#   scope                = azurerm_monitor_workspace.prometheus.id
+#   role_definition_name = "Monitoring Data Reader"
+#   principal_id         = azurerm_dashboard_grafana.grafana.identity.0.principal_id # azurerm_user_assigned_identity.identity-grafana.principal_id # 
+# }
+
+# data "azurerm_subscription" "current" {}
+
+# resource "azurerm_role_assignment" "role_monitoring_reader" {
+#   scope                = data.azurerm_subscription.current.id
+#   role_definition_name = "Monitoring Reader"
+#   principal_id         = azurerm_dashboard_grafana.grafana.identity.0.principal_id # azurerm_user_assigned_identity.identity-grafana.principal_id # 
+# }
+
+# # resource "azurerm_user_assigned_identity" "identity-grafana" {
+# #   name                = "identity-grafana"
+# #   resource_group_name = azurerm_resource_group.rg_monitoring.name
+# #   location            = azurerm_resource_group.rg_monitoring.location
+# # }
diff --git a/85_prometheus_grafana_private_endpoint/output.tf b/85_prometheus_grafana_private_endpoint/output.tf
@@ -1,18 +1,18 @@
-output "query_endpoint" {
-  value = azurerm_monitor_workspace.prometheus.query_endpoint
-}
+# output "query_endpoint" {
+#   value = azurerm_monitor_workspace.prometheus.query_endpoint
+# }
 
-output "garafana_endpoint" {
-  value = azurerm_dashboard_grafana.grafana.endpoint
-}
+# output "garafana_endpoint" {
+#   value = azurerm_dashboard_grafana.grafana.endpoint
+# }
 
-output "grafana_name" {
-  value = azurerm_dashboard_grafana.grafana.name
-}
+# output "grafana_name" {
+#   value = azurerm_dashboard_grafana.grafana.name
+# }
 
-output "grafana_rg_name" {
-  value = azurerm_dashboard_grafana.grafana.resource_group_name
-}
+# output "grafana_rg_name" {
+#   value = azurerm_dashboard_grafana.grafana.resource_group_name
+# }
 
 output "aks_name" {
   value = azurerm_kubernetes_cluster.aks.name

diff --git a/...rivate_endpoint/private_endpoint-ampls.tf → ...heus_grafana_private_endpoint/pe-ampls.tf b/...rivate_endpoint/private_endpoint-ampls.tf → ...heus_grafana_private_endpoint/pe-ampls.tf
@@ -1,3 +1,13 @@
+locals {
+  dns_zones_ampls = [
+    "privatelink.monitor.azure.com",
+    "privatelink.oms.opinsights.azure.com",
+    "privatelink.ods.opinsights.azure.com",
+    "privatelink.agentsvc.azure-automation.net",
+    "privatelink.blob.core.windows.net"
+  ]
+}
+
 resource "azurerm_private_endpoint" "pe-ampls" {
   name                = "pe-ampls"
   resource_group_name = azurerm_virtual_network.vnet.resource_group_name
@@ -13,27 +23,12 @@ resource "azurerm_private_endpoint" "pe-ampls" {
 
   private_dns_zone_group {
     name                 = "private-dns-zone"
-    private_dns_zone_ids = [ for zone in azurerm_private_dns_zone.zones : zone.id ]
+    private_dns_zone_ids = [for zone in azurerm_private_dns_zone.zones : zone.id]
   }
 }
 
-output "zone_id" {
-  value = [ for zone in azurerm_private_dns_zone.zones : zone.id ]
-}
-
-variable "dns_zones_ampls" {
-  type = list(string)
-  default = [
-    "privatelink.monitor.azure.com",
-    "privatelink.oms.opinsights.azure.com",
-    "privatelink.ods.opinsights.azure.com",
-    "privatelink.agentsvc.azure-automation.net",
-    "privatelink.blob.core.windows.net"
-  ]
-}
-
 resource "azurerm_private_dns_zone" "zones" {
-  for_each            = toset(var.dns_zones_ampls)
+  for_each            = toset(local.dns_zones_ampls)
   name                = each.value
   resource_group_name = azurerm_resource_group.rg_monitoring.name
 }