Enable Workload Identity

.github/workflows/test.yml

@@ -46,7 +46,7 @@ on:
         required: false
       region:
         description: "Region (needs to be same as byo vnet location)"
-        default: "eastus2"
+        default: "centralus"
         type: string
         required: false
       doStandards:
@@ -74,7 +74,7 @@ env:
   AZCLIVERSION: 2.63.0    # https://github.com/Azure/azure-cli/issues/29828
   ParamFilePath: ".github/parameters.json"
   DEPNAME: "dep${{ github.run_number }}"
-  
+
 jobs:
   Standards:
     runs-on: ubuntu-latest
@@ -150,7 +150,7 @@ jobs:
           if [ -z "${{ github.event.inputs.region }}" ]
           then
             echo "Region parameter not available through GitHub event data, setting default"
-            REGION="eastus2"
+            REGION="centralus"
           else
             echo "Region parameter found in GitHub event (${{ github.event.inputs.region }})"
             REGION="${{ github.event.inputs.region }}"
@@ -270,8 +270,8 @@ jobs:
           azcliversion: ${{ env.AZCLIVERSION }}
           inlineScript: |
             DEPNAME='Dep${{ github.run_number }}'
-            
-            PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES }} applicationClientId=${{ env.AZURE_CLIENT_ID }} applicationClientSecret=${{ secrets.AZURE_CLIENT_SECRET }} applicationClientPrincipalOid=${{ env.AZURE_CLIENT_PRINCIPAL_OID }} emailAddress=${{ secrets.EMAIL_ADDRESS }}'
+
+            PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES }} applicationClientId=${{ env.AZURE_CLIENT_ID }} applicationClientPrincipalOid=${{ env.AZURE_CLIENT_PRINCIPAL_OID }} emailAddress=${{ secrets.EMAIL_ADDRESS }}'
             echo $PARAMS
             az deployment group validate -f bicep/main.bicep -g $RESOURCE_GROUP -p ${{ env.ParamFilePath }} -p $PARAMS --verbose
 
@@ -330,7 +330,7 @@ jobs:
 
       - name: Install azd
         uses: Azure/[email protected]
-      
+
       - name: Log in with Azure (Federated Credentials)
         if: ${{ env.AZURE_CLIENT_ID != '' }}
         run: |

bicep/main.parameters.json

@@ -14,9 +14,6 @@
     "emailAddress": {
       "value": "${EMAIL_ADDRESS}"
     },
-    "customVMSize": {
-      "value": "${CLUSTER_VM_SIZE}"
-    },
     "ingressType": {
       "value": "${CLUSTER_INGRESS}"
     },
@@ -33,6 +30,13 @@
         "enableLockDown": "${ENABLE_LOCK_DOWN}"
       }
     },
+    "serverConfiguration": {
+      "value": {
+        "systemPool": "${VMSIZE_SYSTEM_POOL}",
+        "zonePool": "${VMSIZE_ZONE_POOL}",
+        "userPool": "${VMSIZE_USER_POOL}"
+      }
+    },
     "vnetConfiguration": {
       "value": {
         "group": "${VIRTUAL_NETWORK_GROUP}",

bicep/modules/blade_cluster.bicep

@@ -17,18 +17,30 @@ param enableTelemetry bool
 @description('The workspace resource Id for diagnostics')
 param workspaceResourceId string
 
-@description('A Custom VM Size for Internal Pool')
-param vmSize string
+// D4pds v5 with 4 vCPUs and 16 GiB of memory. Available in 22 regions starting from $88.18 per month.
+// D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month.
+@description('A Custom VM Size for System Pool (4x8 ARM:true)')
+param vmSizeSystemPool string = 'Standard_D4pds_v6'
+
+// D2pds v5 with 2 vCPUs and 8 GiB of memory. Available in 22 regions starting from $44.09 per month.
+// D2s_v5 with 2 vCPUs and 8 GiB of memory. Available in 50 regions starting from $70.08 per month.
+@description('A Custom VM Size for Zone Pool (2x8 ARM:true)')
+param vmSizeZonePool string = 'Standard_D2pds_v6'
+
+// B4s_v2 with 4 vCPUs and 16 GiB of memory. Available in 49 regions starting from $16.64 per month.
+// D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month.
+@description('A Custom VM Size for User Pool (2x8 ARM:false BURST:true)')
+param vmSizeUserPool string = 'Standard_B4s_v2'
 
 @minLength(9)
 @maxLength(18)
 @description('The address range to use for services')
-param serviceCidr string = '172.16.0.0/16'
+param serviceCidr string = '10.0.0.0/16'
 
 @minLength(7)
 @maxLength(15)
 @description('The IP address to reserve for DNS')
-param dnsServiceIP string = '172.16.0.10'
+param dnsServiceIP string = '10.0.0.10'
 
 @description('The id of the subnet to deploy the AKS nodes')
 param aksSubnetId string
@@ -51,6 +63,8 @@ param enablePrivateCluster bool = true
 @description('Feature Flag to Enable Node Resource Group Lock Down')
 param nodeResourceGroupLockDown bool = true
 
+
+
 /////////////////////////////////
 // Configuration 
 /////////////////////////////////
@@ -61,15 +75,18 @@ var serviceLayerConfig = {
   }
   cluster: {
     tier: 'Standard'
+    sku: 'Base'
     aksVersion: '1.30'
 
-    // D2pds v5 with 2 vCPUs and 8 GiB of memory. Available in 22 regions starting from $44.09 per month.
-    // D4pds v5 with 4 vCPUs and 16 GiB of memory. Available in 22 regions starting from $88.18 per month.
-    // D2s_v5 with 2 vCPUs and 8 GiB of memory. Available in 50 regions starting from $70.08 per month.
-    // D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month.
-    vmSize: 'Standard_D4pds_v5' 
-    poolSize: 'Standard_D2pds_v5'  
-    defaultSize: 'Standard_D4s_v5' // OSDU Java Services don't run on ARM?
+    // // D2pds v5 with 2 vCPUs and 8 GiB of memory. Available in 22 regions starting from $44.09 per month.
+    // // D4pds v5 with 4 vCPUs and 16 GiB of memory. Available in 22 regions starting from $88.18 per month.
+    // // D2s_v5 with 2 vCPUs and 8 GiB of memory. Available in 50 regions starting from $70.08 per month.
+    // // D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month.
+    // // D4ps_v5 with 4 vCPUs and 16 GiB of memory. Available in 23 regions, starting from $73.73 per month.
+    // // B4s_v2 with 4 vCPUs and 16 GiB of memory. Available in 49 regions starting from $16.64 per month.
+    // vmSize: 'Standard_D4pds_v6' 
+    // poolSize: 'Standard_D2pds_v6'  
+    // defaultSize: 'Standard_B4s_v2' // OSDU Java Services don't run on ARM?
   }
 }
 
@@ -96,6 +113,7 @@ module cluster './managed-cluster/main.bicep' = {
     name: '${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}'
     location: location
     skuTier: serviceLayerConfig.cluster.tier
+    skuName: serviceLayerConfig.cluster.sku
     kubernetesVersion: serviceLayerConfig.cluster.aksVersion
 
     // Assign Tags
@@ -209,7 +227,7 @@ module cluster './managed-cluster/main.bicep' = {
       {
         name: 'system'
         mode: 'System'
-        vmSize: empty(vmSize) ? serviceLayerConfig.cluster.vmSize : vmSize
+        vmSize: vmSizeSystemPool
         enableAutoScaling: !enableNodeAutoProvisioning
         count: enableNodeAutoProvisioning ? 2 : null
         minCount: enableNodeAutoProvisioning ? null : 2
@@ -237,7 +255,7 @@ module cluster './managed-cluster/main.bicep' = {
       {
         name: 'default'
         mode: 'User'
-        vmSize: empty(vmSize) ? serviceLayerConfig.cluster.defaultSize : vmSize
+        vmSize: vmSizeUserPool
         enableAutoScaling: !enableNodeAutoProvisioning
         count: enableNodeAutoProvisioning ? 4 : null
         minCount: enableNodeAutoProvisioning ? null : 4
@@ -256,7 +274,7 @@ module cluster './managed-cluster/main.bicep' = {
       {
         name: 'poolz1'
         mode: 'User'
-        vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize
+        vmSize: vmSizeZonePool
         enableAutoScaling: !enableNodeAutoProvisioning
         minCount: enableNodeAutoProvisioning ? null : 1
         maxCount: enableNodeAutoProvisioning ? null : 3
@@ -277,7 +295,7 @@ module cluster './managed-cluster/main.bicep' = {
       {
         name: 'poolz2'
         mode: 'User'
-        vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize
+        vmSize: vmSizeZonePool
         enableAutoScaling: !enableNodeAutoProvisioning
         minCount: enableNodeAutoProvisioning ? null : 1
         maxCount: enableNodeAutoProvisioning ? null : 3
@@ -298,7 +316,7 @@ module cluster './managed-cluster/main.bicep' = {
       {
         name: 'poolz3'
         mode: 'User'
-        vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize
+        vmSize: vmSizeZonePool
         enableAutoScaling: !enableNodeAutoProvisioning
         minCount: enableNodeAutoProvisioning ? null : 1
         maxCount: enableNodeAutoProvisioning ? null : 3
@@ -394,6 +412,9 @@ output natClusterIP string = natClusterIP.outputs.ipAddress
 @description('The OIDC Issuer URL for the cluster.')
 output oidcIssuerUrl string = cluster.outputs.oidcIssuerUrl
 
+@description('The Object ID of the Kubelet Identity.') 
+output kubeletIdentityId string = cluster.outputs.kubeletIdentityObjectId
+
 // =============== //
 //   Definitions   //
 // =============== //

bicep/modules/blade_configuration.bicep

@@ -1,5 +1,5 @@
 /////////////////
-// Configuration Blade 
+// Configuration Blade
 /////////////////
 
 @description('The configuration for the blade section.')
@@ -12,10 +12,10 @@ param location string
 param tags object = {}
 
 @description('The name of the Key Vault where the secret exists')
-param kvName string 
+param kvName string
 
 @description('The Uri of the Key Vault where the secret exists')
-param kvUri string 
+param kvUri string
 
 @description('The name of the cluster.')
 param clusterName string
@@ -113,12 +113,21 @@ resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = {
   name: kvName
 }
 
-resource keySecret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = {
+resource keySecretSpUsername 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = {
   name: 'app-dev-sp-username'
   parent: keyVault
 
   properties: {
-    value: applicationClientId
+    value: appIdentity.properties.clientId
+  }
+}
+
+resource keySecretSpPassword 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = {
+  name: 'app-dev-sp-password'
+  parent: keyVault
+
+  properties: {
+    value: 'dummy'
   }
 }
 
@@ -169,12 +178,12 @@ var federatedIdentityCredentials = [
 ]
 
 /*
- _______  _______  _______   _______ .______          ___   .___________. __    ______   .__   __. 
-|   ____||   ____||       \ |   ____||   _  \        /   \  |           ||  |  /  __  \  |  \ |  | 
-|  |__   |  |__   |  .--.  ||  |__   |  |_)  |      /  ^  \ `---|  |----`|  | |  |  |  | |   \|  | 
-|   __|  |   __|  |  |  |  ||   __|  |      /      /  /_\  \    |  |     |  | |  |  |  | |  . `  | 
-|  |     |  |____ |  '--'  ||  |____ |  |\  \----./  _____  \   |  |     |  | |  `--'  | |  |\   | 
-|__|     |_______||_______/ |_______|| _| `._____/__/     \__\  |__|     |__|  \______/  |__| \__| 
+ _______  _______  _______   _______ .______          ___   .___________. __    ______   .__   __.
+|   ____||   ____||       \ |   ____||   _  \        /   \  |           ||  |  /  __  \  |  \ |  |
+|  |__   |  |__   |  .--.  ||  |__   |  |_)  |      /  ^  \ `---|  |----`|  | |  |  |  | |   \|  |
+|   __|  |   __|  |  |  |  ||   __|  |      /      /  /_\  \    |  |     |  | |  |  |  | |  . `  |
+|  |     |  |____ |  '--'  ||  |____ |  |\  \----./  _____  \   |  |     |  | |  `--'  | |  |\   |
+|__|     |_______||_______/ |_______|| _| `._____/__/     \__\  |__|     |__|  \______/  |__| \__|
 */
 @batchSize(1)
 module federatedCredentials './federated_identity.bicep' = [for (cred, index) in federatedIdentityCredentials: {
@@ -207,6 +216,12 @@ var common_helm_values = [
     contentType: 'text/plain'
     label: 'configmap-common-values'
   }
+  {
+    name: 'AZURE_PAAS_WORKLOADIDENTITY_ISENABLED'
+    value: 'true'
+    contentType: 'text/plain'
+    label: 'configmap-common-values'
+  }
   {
     name: 'ACCEPT_HTTP'
     value: 'true'
@@ -323,12 +338,12 @@ var partitionStorageSettings = [for (name, i) in partitionStorageNames: {
 }]
 
 /*
-     ___      .______   .______     ______   ______   .__   __.  _______  __    _______ 
+     ___      .______   .______     ______   ______   .__   __.  _______  __    _______
     /   \     |   _  \  |   _  \   /      | /  __  \  |  \ |  | |   ____||  |  /  _____|
-   /  ^  \    |  |_)  | |  |_)  | |  ,----'|  |  |  | |   \|  | |  |__   |  | |  |  __  
-  /  /_\  \   |   ___/  |   ___/  |  |     |  |  |  | |  . `  | |   __|  |  | |  | |_ | 
- /  _____  \  |  |      |  |      |  `----.|  `--'  | |  |\   | |  |     |  | |  |__| | 
-/__/     \__\ | _|      | _|       \______| \______/  |__| \__| |__|     |__|  \______| 
+   /  ^  \    |  |_)  | |  |_)  | |  ,----'|  |  |  | |   \|  | |  |__   |  | |  |  __
+  /  /_\  \   |   ___/  |   ___/  |  |     |  |  |  | |  . `  | |   __|  |  | |  | |_ |
+ /  _____  \  |  |      |  |      |  `----.|  `--'  | |  |\   | |  |     |  | |  |__| |
+/__/     \__\ | _|      | _|       \______| \______/  |__| \__| |__|     |__|  \______|
 */
 // AVM Module Customized due for east of settings.
 module app_config './app-configuration/main.bicep' = {
@@ -397,12 +412,12 @@ values.yaml: |
 }
 
 /*
-  ______   ______   .__   __.  _______  __    _______ .___  ___.      ___      .______   
- /      | /  __  \  |  \ |  | |   ____||  |  /  _____||   \/   |     /   \     |   _  \  
-|  ,----'|  |  |  | |   \|  | |  |__   |  | |  |  __  |  \  /  |    /  ^  \    |  |_)  | 
-|  |     |  |  |  | |  . `  | |   __|  |  | |  | |_ | |  |\/|  |   /  /_\  \   |   ___/  
-|  `----.|  `--'  | |  |\   | |  |     |  | |  |__| | |  |  |  |  /  _____  \  |  |      
- \______| \______/  |__| \__| |__|     |__|  \______| |__|  |__| /__/     \__\ | _|      
+  ______   ______   .__   __.  _______  __    _______ .___  ___.      ___      .______
+ /      | /  __  \  |  \ |  | |   ____||  |  /  _____||   \/   |     /   \     |   _  \
+|  ,----'|  |  |  | |   \|  | |  |__   |  | |  |  __  |  \  /  |    /  ^  \    |  |_)  |
+|  |     |  |  |  | |  . `  | |   __|  |  | |  | |_ | |  |\/|  |   /  /_\  \   |   ___/
+|  `----.|  `--'  | |  |\   | |  |     |  | |  |__| | |  |  |  |  /  _____  \  |  |
+ \______| \______/  |__| \__| |__|     |__|  \______| |__|  |__| /__/     \__\ | _|
 */
 module appConfigMap './aks-config-map/main.bicep' = {
   name: '${bladeConfig.sectionName}-cluster-appconfig-configmap'
@@ -411,16 +426,16 @@ module appConfigMap './aks-config-map/main.bicep' = {
     location: location
     name: 'config-map-values'
     namespace: 'default'
-    
+
     newOrExistingManagedIdentity: 'existing'
     managedIdentityName: managedIdentityName
     existingManagedIdentitySubId: subscription().subscriptionId
     existingManagedIdentityResourceGroupName:resourceGroup().name
 
     // Order of items matters here.
     fileData: [
-      format(configMaps.appConfigTemplate, 
-             subscription().tenantId, 
+      format(configMaps.appConfigTemplate,
+             subscription().tenantId,
              appIdentity.properties.clientId,
              app_config.outputs.endpoint,
              kvUri,
@@ -457,9 +472,9 @@ var serviceLayerConfig = {
 /* _______  __  .___________.  ______   .______     _______.
  /  _____||  | |           | /  __  \  |   _  \   /       |
 |  |  __  |  | `---|  |----`|  |  |  | |  |_)  | |   (----`
-|  | |_ | |  |     |  |     |  |  |  | |   ___/   \   \    
-|  |__| | |  |     |  |     |  `--'  | |  |   .----)   |   
- \______| |__|     |__|      \______/  | _|   |_______/                                                          
+|  | |_ | |  |     |  |     |  |  |  | |   ___/   \   \
+|  |__| | |  |     |  |     |  `--'  | |  |   .----)   |
+ \______| |__|     |__|      \______/  | _|   |_______/
 */
 //--------------Flux Config---------------
 module fluxConfiguration 'br/public:avm/res/kubernetes-configuration/flux-configuration:0.3.3' = if(enableSoftwareLoad) {