From 3314cc92375d9e84ccbaafa0806925a0f9a72e77 Mon Sep 17 00:00:00 2001
From: Bengt <32734566+bengtfredh@users.noreply.github.com>
Date: Fri, 25 Oct 2024 10:05:56 +0200
Subject: [PATCH] add iac admin-test-aks-rg (#1012)

* deployed admin-test-aks

* add gh action admin-test-aks

* update flux-config admin-test-aks

* data altinncr

---------

Co-authored-by: Bengt Fredh <bengt.rino.fredh@digdir.no>
---
 .../workflows/admin-test-aks-rg-deploy.yml    |  86 +++++++++++++++
 .../admin-test-aks-rg/aks.tf                  | 104 ++++++++++++++++++
 .../admin-test-aks-rg/data.tf                 |   5 +
 .../admin-test-aks-rg/flux-config.tf          |  37 +++++++
 .../admin-test-aks-rg/flux.tf                 |  11 ++
 .../admin-test-aks-rg/metrics-server-cm.tf    |  21 ++++
 .../admin-test-aks-rg/network.tf              |  56 ++++++++++
 .../admin-test-aks-rg/providers.tf            |  42 +++++++
 .../admin-test-aks-rg/rg.tf                   |   4 +
 .../admin-test-aks-rg/terraform.tfvars        |  22 ++++
 .../admin-test-aks-rg/traefik-helm-cm.tf      |  27 +++++
 .../admin-test-aks-rg/variables.tf            |  32 ++++++
 12 files changed, 447 insertions(+)
 create mode 100644 .github/workflows/admin-test-aks-rg-deploy.yml
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/aks.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/data.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/flux-config.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/flux.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/metrics-server-cm.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/network.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/providers.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/rg.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/terraform.tfvars
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/traefik-helm-cm.tf
 create mode 100644 infrastructure/adminservices-test/admin-test-aks-rg/variables.tf

diff --git a/.github/workflows/admin-test-aks-rg-deploy.yml b/.github/workflows/admin-test-aks-rg-deploy.yml
new file mode 100644
index 00000000..a53ece64
--- /dev/null
+++ b/.github/workflows/admin-test-aks-rg-deploy.yml
@@ -0,0 +1,86 @@
+name: admin-test-aks-rg deploy
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/admin-test-aks-rg-deploy.yml
+      - actions/terraform/apply/**
+      - actions/terraform/plan/**
+      - infrastructure/adminservices-test/altinn-monitor-test-rg/**
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/workflows/admin-test-aks-rg-deploy.yml
+      - actions/terraform/apply/**
+      - actions/terraform/plan/**
+      - infrastructure/adminservices-test/admin-test-aks-rg/**
+  workflow_dispatch:
+    inputs:
+      log_level:
+        required: true
+        description: Terraform Log Level
+        default: INFO
+        type: choice
+        options:
+          - TRACE
+          - DEBUG
+          - INFO
+          - WARN
+          - ERROR
+
+env:
+  ENVIRONMENT: test
+  TF_STATE_NAME: admin-test-aks-rg.tfstate
+  TF_PROJECT: ./infrastructure/adminservices-test/admin-test-aks-rg
+  ARM_CLIENT_ID: ${{ vars.TF_AZURE_CLIENT_ID }}
+  ARM_SUBSCRIPTION_ID: 1ce8e9af-c2d6-44e7-9c5e-099a308056fe
+
+permissions:
+  id-token: write
+  contents: write
+  pull-requests: write
+
+jobs:
+  plan:
+    name: Plan
+    environment: reader
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Terraform Plan
+        uses: altinn/altinn-platform/actions/terraform/plan@main
+        with:
+          working_directory: ${{ env.TF_PROJECT }}
+          oidc_type: environment
+          oidc_value: ${{ env.ENVIRONMENT }}
+          arm_client_id: ${{ env.ARM_CLIENT_ID }}
+          arm_subscription_id: ${{ env.ARM_SUBSCRIPTION_ID }}
+          tf_state_name: ${{ env.TF_STATE_NAME }}
+          gh_token: ${{ secrets.GITHUB_TOKEN }}
+          tf_version: latest
+
+  deploy:
+    name: Deploy
+    environment: test
+    if: github.ref == 'refs/heads/main'
+    needs: plan
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Terraform Apply
+        uses: altinn/altinn-platform/actions/terraform/apply@main
+        with:
+          working_directory: ${{ env.TF_PROJECT }}
+          oidc_type: environment
+          oidc_value: ${{ env.ENVIRONMENT }}
+          arm_client_id: ${{ env.ARM_CLIENT_ID }}
+          arm_subscription_id: ${{ env.ARM_SUBSCRIPTION_ID }}
+          tf_state_name: ${{ env.TF_STATE_NAME }}
+          tf_version: latest
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/aks.tf b/infrastructure/adminservices-test/admin-test-aks-rg/aks.tf
new file mode 100644
index 00000000..701476db
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/aks.tf
@@ -0,0 +1,104 @@
+resource "azurerm_kubernetes_cluster" "aks" {
+  name                      = "${var.name_prefix}-aks"
+  location                  = azurerm_resource_group.rg.location
+  resource_group_name       = azurerm_resource_group.rg.name
+  dns_prefix                = var.name_prefix
+  sku_tier                  = var.aks_sku_tier
+  kubernetes_version        = var.kubernetes_version
+  automatic_upgrade_channel = "patch"
+  node_os_upgrade_channel   = "NodeImage"
+  oidc_issuer_enabled       = true
+  workload_identity_enabled = true
+
+  default_node_pool {
+    name                         = "syspool"
+    os_sku                       = "AzureLinux"
+    orchestrator_version         = var.kubernetes_version
+    vnet_subnet_id               = azurerm_subnet.subnets["aks_syspool"].id
+    only_critical_addons_enabled = true
+    temporary_name_for_rotation  = "syspool99"
+    auto_scaling_enabled         = true
+    max_pods                     = 200
+    vm_size                      = var.pool_configs["syspool"].vm_size
+    min_count                    = var.pool_configs["syspool"].min_count
+    max_count                    = var.pool_configs["syspool"].max_count
+    zones                        = ["1", "2", "3"]
+    upgrade_settings {
+      max_surge = "10%"
+    }
+  }
+
+  network_profile {
+    network_plugin      = "azure"
+    network_plugin_mode = "overlay"
+    ip_versions         = ["IPv4", "IPv6"] # Azure did not like IPv6 first
+    pod_cidrs           = ["10.240.0.0/16", "fd10:59f0:8c79:240::/64"]
+    service_cidrs       = ["10.250.0.0/16", "fd10:59f0:8c79:250::/108"]
+    dns_service_ip      = "10.250.0.53"
+    load_balancer_profile {
+      outbound_ip_prefix_ids = [
+        azurerm_public_ip_prefix.prefix4.id,
+        azurerm_public_ip_prefix.prefix6.id
+      ]
+    }
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  monitor_metrics {}
+
+  maintenance_window_auto_upgrade {
+    frequency   = "Weekly"
+    interval    = "1"
+    duration    = "5"
+    day_of_week = "Monday"
+    start_time  = "23:30"
+    utc_offset  = "+00:00"
+  }
+  maintenance_window_node_os {
+    frequency   = "Weekly"
+    interval    = "1"
+    duration    = "5"
+    day_of_week = "Tuesday"
+    start_time  = "23:30"
+    utc_offset  = "+00:00"
+  }
+}
+
+resource "azurerm_kubernetes_cluster_node_pool" "workpool" {
+  lifecycle {
+    ignore_changes = [
+      node_count,
+    ]
+  }
+  name                  = "workpool"
+  os_sku                = "AzureLinux"
+  kubernetes_cluster_id = azurerm_kubernetes_cluster.aks.id
+  vnet_subnet_id        = azurerm_subnet.subnets["aks_workpool"].id
+  orchestrator_version  = var.kubernetes_version
+  auto_scaling_enabled  = true
+  max_pods              = 200
+  vm_size               = var.pool_configs["workpool"].vm_size
+  min_count             = var.pool_configs["workpool"].min_count
+  max_count             = var.pool_configs["workpool"].max_count
+  zones                 = ["1", "2", "3"]
+  upgrade_settings {
+    max_surge = "10%"
+  }
+}
+
+resource "azurerm_role_assignment" "aks_id_rg_contributor" {
+  scope                            = azurerm_resource_group.rg.id
+  role_definition_name             = "Contributor"
+  principal_id                     = azurerm_kubernetes_cluster.aks.identity[0].principal_id
+  skip_service_principal_aad_check = true
+}
+
+resource "azurerm_role_assignment" "altinncr_acrpull" {
+  principal_id                     = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id
+  role_definition_name             = "AcrPull"
+  scope                            = data.azurerm_container_registry.altinncr.id
+  skip_service_principal_aad_check = true
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/data.tf b/infrastructure/adminservices-test/admin-test-aks-rg/data.tf
new file mode 100644
index 00000000..82ef938d
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/data.tf
@@ -0,0 +1,5 @@
+data "azurerm_container_registry" "altinncr" {
+  provider            = azurerm.adminservices-prod
+  name                = "altinncr"
+  resource_group_name = "acr"
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/flux-config.tf b/infrastructure/adminservices-test/admin-test-aks-rg/flux-config.tf
new file mode 100644
index 00000000..5a627726
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/flux-config.tf
@@ -0,0 +1,37 @@
+resource "kubectl_manifest" "flux_config" {
+  depends_on = [azurerm_kubernetes_cluster_extension.flux_ext]
+  yaml_body  = <<YAML
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: OCIRepository
+metadata:
+  name: flux-config
+  namespace: flux-system
+spec:
+  interval: 5m
+  url: oci://altinncr.azurecr.io/manifests/config
+  provider: azure
+  ref:
+    tag: admin-test
+YAML
+}
+
+resource "kubectl_manifest" "flux_config_kustomize" {
+  depends_on = [azurerm_kubernetes_cluster_extension.flux_ext]
+  yaml_body  = <<YAML
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: flux-config
+  namespace: flux-system
+spec:
+  sourceRef:
+    kind: OCIRepository
+    name: flux-config
+  interval: 5m
+  retryInterval: 5m
+  path: ./
+  prune: true
+  wait: true
+  timeout: 2m
+YAML
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/flux.tf b/infrastructure/adminservices-test/admin-test-aks-rg/flux.tf
new file mode 100644
index 00000000..21c665f6
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/flux.tf
@@ -0,0 +1,11 @@
+resource "azurerm_kubernetes_cluster_extension" "flux_ext" {
+  depends_on     = [azurerm_kubernetes_cluster.aks]
+  name           = "flux-ext"
+  cluster_id     = azurerm_kubernetes_cluster.aks.id
+  extension_type = "microsoft.flux"
+  configuration_settings = {
+    "useKubeletIdentity"      = "true"
+    "autoUpgradeMinorVersion" = "true"
+    "multiTenancy.enforce"    = "false"
+  }
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/metrics-server-cm.tf b/infrastructure/adminservices-test/admin-test-aks-rg/metrics-server-cm.tf
new file mode 100644
index 00000000..f7d3e9d0
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/metrics-server-cm.tf
@@ -0,0 +1,21 @@
+resource "kubectl_manifest" "metrics_server_configmap" {
+  depends_on = [azurerm_kubernetes_cluster.aks]
+  yaml_body  = <<YAML
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metrics-server-config
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: EnsureExists
+data:
+  NannyConfiguration: |-
+    apiVersion: nannyconfig/v1alpha1
+    kind: NannyConfiguration
+    baseCPU: 100m
+    cpuPerNode: 1m
+    baseMemory: 100Mi
+    memoryPerNode: 8Mi
+YAML
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/network.tf b/infrastructure/adminservices-test/admin-test-aks-rg/network.tf
new file mode 100644
index 00000000..cabebb7f
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/network.tf
@@ -0,0 +1,56 @@
+resource "azurerm_virtual_network" "vnet" {
+  name                = "${var.name_prefix}-vnet"
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
+  address_space       = var.vnet_address_space
+}
+
+resource "azurerm_subnet" "subnets" {
+  for_each             = var.subnet_address_prefixes
+  name                 = each.key
+  resource_group_name  = azurerm_resource_group.rg.name
+  virtual_network_name = azurerm_virtual_network.vnet.name
+  address_prefixes     = each.value
+}
+
+resource "azurerm_public_ip" "pip4" {
+  name                 = "${var.name_prefix}-pip4"
+  location             = azurerm_resource_group.rg.location
+  resource_group_name  = azurerm_kubernetes_cluster.aks.node_resource_group
+  allocation_method    = "Static"
+  zones                = ["1", "2", "3"]
+  ddos_protection_mode = "Enabled"
+  sku                  = "Standard"
+  ip_version           = "IPv4"
+  domain_name_label    = var.name_prefix
+}
+
+resource "azurerm_public_ip" "pip6" {
+  name                 = "${var.name_prefix}-pip6"
+  location             = azurerm_resource_group.rg.location
+  resource_group_name  = azurerm_kubernetes_cluster.aks.node_resource_group
+  allocation_method    = "Static"
+  zones                = ["1", "2", "3"]
+  ddos_protection_mode = "Enabled"
+  sku                  = "Standard"
+  ip_version           = "IPv6"
+  domain_name_label    = var.name_prefix
+}
+
+resource "azurerm_public_ip_prefix" "prefix4" {
+  name                = "${var.name_prefix}-prefix4"
+  resource_group_name = azurerm_resource_group.rg.name
+  location            = azurerm_resource_group.rg.location
+  ip_version          = "IPv4"
+  prefix_length       = "31"
+  zones               = ["1", "2", "3"]
+}
+
+resource "azurerm_public_ip_prefix" "prefix6" {
+  name                = "${var.name_prefix}-prefix6"
+  resource_group_name = azurerm_resource_group.rg.name
+  location            = azurerm_resource_group.rg.location
+  ip_version          = "IPv6"
+  prefix_length       = "127"
+  zones               = ["1", "2", "3"]
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/providers.tf b/infrastructure/adminservices-test/admin-test-aks-rg/providers.tf
new file mode 100644
index 00000000..d0b17009
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/providers.tf
@@ -0,0 +1,42 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~> 4.0"
+    }
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = "~> 1.14.0"
+    }
+  }
+  backend "azurerm" {
+    use_azuread_auth = true
+  }
+}
+
+provider "azurerm" {
+  subscription_id = var.subscription_id
+  features {}
+  resource_providers_to_register = [
+    "Microsoft.Monitor",
+    "Microsoft.AlertsManagement",
+    "Microsoft.Dashboard",
+    "Microsoft.KubernetesConfiguration"
+  ]
+}
+
+provider "azurerm" {
+  alias                           = "adminservices-prod"
+  resource_provider_registrations = "none"
+  subscription_id                 = var.admin_services_prod_subscription_id
+  use_oidc                        = true
+  features {}
+}
+
+provider "kubectl" {
+  load_config_file       = false
+  client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
+  client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
+  host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
+  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/rg.tf b/infrastructure/adminservices-test/admin-test-aks-rg/rg.tf
new file mode 100644
index 00000000..c2f0066d
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/rg.tf
@@ -0,0 +1,4 @@
+resource "azurerm_resource_group" "rg" {
+  name     = "${var.name_prefix}-rg"
+  location = var.location
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/terraform.tfvars b/infrastructure/adminservices-test/admin-test-aks-rg/terraform.tfvars
new file mode 100644
index 00000000..7f5c8aa7
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/terraform.tfvars
@@ -0,0 +1,22 @@
+subscription_id                     = "1ce8e9af-c2d6-44e7-9c5e-099a308056fe"
+admin_services_prod_subscription_id = "a6e9ee7d-2b65-41e1-adfb-0c8c23515cf9"
+name_prefix                         = "admin-test"
+vnet_address_space                  = ["10.90.0.0/16", "fdac:524d:afaf::/56"]
+subnet_address_prefixes = {
+  aks_syspool  = ["fdac:524d:afaf:1::/64", "10.90.1.0/24"]
+  aks_workpool = ["fdac:524d:afaf:2::/64", "10.90.2.0/24"]
+}
+pool_configs = {
+  syspool = {
+    vm_size   = "standard_b2s_v2"
+    min_count = "1"
+    max_count = "3"
+  }
+  workpool = {
+    vm_size   = "standard_b2s_v2"
+    min_count = "0"
+    max_count = "6"
+  }
+}
+kubernetes_version = "1.29"
+aks_sku_tier       = "Free"
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/traefik-helm-cm.tf b/infrastructure/adminservices-test/admin-test-aks-rg/traefik-helm-cm.tf
new file mode 100644
index 00000000..ec44a9a0
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/traefik-helm-cm.tf
@@ -0,0 +1,27 @@
+resource "kubectl_manifest" "traefik_namespace" {
+  depends_on = [azurerm_kubernetes_cluster_extension.flux_ext]
+  yaml_body  = <<YAML
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: traefik
+YAML
+}
+
+resource "kubectl_manifest" "traefik_helm_values_configmap" {
+  depends_on = [azurerm_kubernetes_cluster_extension.flux_ext]
+  yaml_body  = <<YAML
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: traefik-helm-values
+  namespace: traefik
+data:
+  values.yaml: |
+    service:
+      annotations:
+        service.beta.kubernetes.io/azure-load-balancer-resource-group: ${azurerm_kubernetes_cluster.aks.node_resource_group}
+        service.beta.kubernetes.io/azure-load-balancer-ipv4: ${azurerm_public_ip.pip4.ip_address}
+        service.beta.kubernetes.io/azure-load-balancer-ipv6: ${azurerm_public_ip.pip6.ip_address}
+YAML
+}
diff --git a/infrastructure/adminservices-test/admin-test-aks-rg/variables.tf b/infrastructure/adminservices-test/admin-test-aks-rg/variables.tf
new file mode 100644
index 00000000..7e17dcf8
--- /dev/null
+++ b/infrastructure/adminservices-test/admin-test-aks-rg/variables.tf
@@ -0,0 +1,32 @@
+variable "subscription_id" {
+  type = string
+}
+variable "admin_services_prod_subscription_id" {
+  type = string
+}
+variable "location" {
+  type    = string
+  default = "norwayeast"
+}
+variable "name_prefix" {
+  type = string
+}
+variable "vnet_address_space" {
+  type = list(string)
+}
+variable "subnet_address_prefixes" {
+  type = map(list(string))
+}
+variable "kubernetes_version" {
+  type = string
+}
+variable "aks_sku_tier" {
+  type = string
+}
+variable "pool_configs" {
+  type = map(object({
+    vm_size   = string
+    min_count = string
+    max_count = string
+  }))
+}