diff --git a/bicep/README.md b/bicep/README.md index bfcc36fc..07ee9ae0 100644 --- a/bicep/README.md +++ b/bicep/README.md @@ -7,8 +7,6 @@ graph TD MonitoringResources["Monitoring Resources: logAnalytics"] ConditionalNetwork["Network Resources: Conditional Deployments"] ClusterNSG["clusterNetworkSecurityGroup - !vnetInjection"] - BastionNSG["bastionNetworkSecurityGroup - !vnetInjection and enableBastion"] - MachineNSG["machineNetworkSecurityGroup - !vnetInjection and enableBastion"] Network["network - !vnetInjection"] CommonResources["Common Resources"] AppInsights["appInsights"] @@ -20,9 +18,6 @@ graph TD ScriptFileShares["scriptFileShares"] CommonDatabase["commonDatabase"] RedisCache["redisCache"] - ManageResources["Manage Resources"] - BastionHost["bastionHost - enableBastion"] - VirtualMachine["virtualMachine - enableBastion"] PartitionResources["Partition Resources"] PartitionStorage["partitionStorage"] PartitionDatabase["partitionDatabase"] @@ -48,8 +43,6 @@ graph TD IdentityResources --> MonitoringResources MonitoringResources --> ConditionalNetwork ConditionalNetwork -->|"!vnetInjection"| ClusterNSG - ConditionalNetwork -->|"!vnetInjection and enableBastion"| BastionNSG - ConditionalNetwork -->|"!vnetInjection and enableBastion"| MachineNSG ConditionalNetwork -->|"!vnetInjection"| Network ClusterNSG --> CommonResources BastionNSG --> CommonResources @@ -64,9 +57,6 @@ graph TD CommonStorage --> ScriptFileShares KeyVault --> CommonDatabase CommonResources --> RedisCache - CommonResources --> ManageResources - ManageResources -->|"enableBastion"| BastionHost - BastionHost --> |"enableBastion"| VirtualMachine CommonResources --> PartitionResources PartitionResources --> PartitionStorage PartitionResources --> PartitionDatabase diff --git a/bicep/main.bicep b/bicep/main.bicep index 64bbdec5..069d109f 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -17,10 +17,7 @@ param applicationClientSecret string @description('Specify the Enterprise Application Object Id. (This is the unique ID of the service principal object associated with the application.)') param applicationClientPrincipalOid string -@description('Feature Flag: Enable Burstable Server Types') -param enableBurstable bool = false - -@description('Use customized server types.') +@description('The size of the VM to use for the cluster.') param customVMSize string = '' @allowed([ @@ -35,12 +32,6 @@ param ingressType string = 'External' @description('Feature Flag: Enable Storage accounts public access.') param enableBlobPublicAccess bool = false -@description('Feature Flag: Enable management with a virtual machine and bastion host.') -param enableManage bool = false - -@description('(Optional) If manage then the ssh user name for the virtual machine.') -param vmAdminUsername string = 'azureUser' - @description('Feature Flag: Enable AKS Enhanced Subnet Support (Azure CNI)') param enablePodSubnet bool = false @@ -88,6 +79,7 @@ param experimentalSoftware object = { adminUI: false } + // This would be a type but bugs exist for ARM Templates so is object instead. @description('Cluster Network Overrides - {ingress} (Both/Internal/External), {serviceCidr}, {dnsServiceIP}') param clusterNetwork object = { @@ -96,16 +88,6 @@ param clusterNetwork object = { dnsServiceIP: '' } -@allowed([ - 'kubenet' - 'azure' -]) -@description('The network plugin to use for the Kubernetes cluster.') -param clusterNetworkPlugin string = 'azure' - -@description('Optional: Specify the AD Users and/or Groups that can manage the cluster.') -param clusterAdminIds array = [] - ///////////////////////////////// // Configuration ///////////////////////////////// @@ -210,7 +192,7 @@ module logAnalytics 'br/public:avm/res/operational-insights/workspace:0.3.4' = { //*****************************************************************// // Network Resources // //*****************************************************************// -module networkBlade 'modules/blade_network.bicep' = { +module networkBlade 'modules/blade_network.bicep' = if (enableVnetInjection) { name: 'network-blade' params: { bladeConfig: { @@ -228,7 +210,6 @@ module networkBlade 'modules/blade_network.bicep' = { workspaceResourceId: logAnalytics.outputs.resourceId identityId: stampIdentity.outputs.principalId - enableBastion: enableManage enablePodSubnet: enablePodSubnet enableVnetInjection: enableVnetInjection @@ -281,10 +262,12 @@ module commonBlade 'modules/blade_common.bicep' = { enableTelemetry: enableTelemetry deploymentScriptIdentity: stampIdentity.outputs.name + userAssignedIdentityName: stampIdentity.outputs.name + workspaceResourceId: logAnalytics.outputs.resourceId workspaceName: logAnalytics.outputs.name - subnetId: networkBlade.outputs.aksSubnetId + subnetId: enableVnetInjection ? networkBlade.outputs.aksSubnetId : '' cmekConfiguration: cmekConfiguration enablePrivateLink: enablePrivateLink @@ -294,60 +277,11 @@ module commonBlade 'modules/blade_common.bicep' = { applicationClientSecret: applicationClientSecret applicationClientPrincipalOid: applicationClientPrincipalOid } - dependsOn: [ - networkBlade - ] -} - - -//*****************************************************************// -// Manage Resources // -//*****************************************************************// -module manageBlade 'modules/blade_manage.bicep' = { - name: 'manage-blade' - params: { - bladeConfig: { - sectionName: 'manageblade' - displayName: 'Manage Resources' - } - - tags: { - id: rg_unique_id - } - - manageLayerConfig: { - machine: { - vmSize: 'Standard_DS3_v2' - imagePublisher: 'Canonical' - imageOffer: 'UbuntuServer' - imageSku: '18.04-LTS' - authenticationType: 'password' - } - bastion: { - skuName: 'Basic' - } - } - - location: location - enableTelemetry: enableTelemetry - - workspaceName: logAnalytics.outputs.name - kvName: commonBlade.outputs.keyvaultName - - // Feature Flags - enableBastion: enableManage - - vmAdminUsername: vmAdminUsername - vnetId: networkBlade.outputs.vnetId - vmSubnetId: networkBlade.outputs.vmSubnetId - } - dependsOn: [ + dependsOn: enableVnetInjection ? [ networkBlade - commonBlade - ] + ] :[] } - //*****************************************************************// // Partition Resources // //*****************************************************************// @@ -367,7 +301,7 @@ module partitionBlade 'modules/blade_partition.bicep' = { workspaceResourceId: logAnalytics.outputs.resourceId kvName: commonBlade.outputs.keyvaultName - subnetId: networkBlade.outputs.aksSubnetId + subnetId: enableVnetInjection ? networkBlade.outputs.aksSubnetId : '' enableBlobPublicAccess: enableBlobPublicAccess enablePrivateLink: enablePrivateLink @@ -375,13 +309,14 @@ module partitionBlade 'modules/blade_partition.bicep' = { storageDNSZoneId: commonBlade.outputs.storageDNSZoneId cosmosDNSZoneId: commonBlade.outputs.cosmosDNSZoneId - partitionSize: enableBurstable ? 'Burstable' : 'Standard' partitions: configuration.partitions managedIdentityName: stampIdentity.outputs.name } - dependsOn: [ + dependsOn: enableVnetInjection ? [ networkBlade commonBlade + ] :[ + commonBlade ] } @@ -424,16 +359,13 @@ module serviceBlade 'modules/blade_service.bicep' = { partitionStorageNames: partitionBlade.outputs.partitionStorageNames partitionServiceBusNames: partitionBlade.outputs.partitionServiceBusNames - aksSubnetId: networkBlade.outputs.aksSubnetId - podSubnetId: enablePodSubnet ? networkBlade.outputs.podSubnetId : '' - customVMSize: customVMSize - clusterSize: enableBurstable ? 'Burstable' : 'Standard' - clusterAdminIds: clusterAdminIds + aksSubnetId: enableVnetInjection ? networkBlade.outputs.aksSubnetId : '' + podSubnetId: enableVnetInjection && enablePodSubnet ? networkBlade.outputs.podSubnetId : '' + vmSize: customVMSize clusterIngress: ingressType == '' ? 'External' : ingressType serviceCidr: clusterNetwork.serviceCidr == '' ? '172.16.0.0/16' : clusterNetwork.serviceCidr - dnsServiceIP: clusterNetwork.dnsServiceIP == '' ? '172.16.0.10' : clusterNetwork.v - networkPlugin: enablePodSubnet ? 'azure' : clusterNetworkPlugin + dnsServiceIP: clusterNetwork.dnsServiceIP == '' ? '172.16.0.10' : clusterNetwork.vnet softwareBranch: clusterSoftware.branch softwareRepository: clusterSoftware.repository @@ -455,7 +387,6 @@ module serviceBlade 'modules/blade_service.bicep' = { ] } dependsOn: [ - networkBlade commonBlade partitionBlade ] diff --git a/bicep/main.parameters.json b/bicep/main.parameters.json index 39902517..40731927 100644 --- a/bicep/main.parameters.json +++ b/bicep/main.parameters.json @@ -14,9 +14,6 @@ "emailAddress": { "value": "${EMAIL_ADDRESS}" }, - "enableBurstable": { - "value": "${ENABLE_BURSTABLE}" - }, "customVMSize": { "value": "${CLUSTER_VM_SIZE}" }, diff --git a/bicep/modules/aks_agent_pool.bicep b/bicep/modules/aks_agent_pool.bicep deleted file mode 100644 index b80786a4..00000000 --- a/bicep/modules/aks_agent_pool.bicep +++ /dev/null @@ -1,75 +0,0 @@ -param AksName string - -param PoolName string - -@description('The zones to use for a node pool') -param availabilityZones array = [] - -@description('OS disk type') -param osDiskType string = 'Managed' - -@description('VM SKU') -param agentVMSize string = 'Standard_DS3_v2' - -@description('Disk size in GB') -param osDiskSizeGB int = 0 - -@description('The number of agents for the user node pool') -param agentCount int = 3 - -@description('The maximum number of nodes for the user node pool') -param agentCountMax int = 0 -var autoScale = agentCountMax > agentCount - -@description('The maximum number of pods per node.') -param maxPods int = 30 - -@description('Any taints that should be applied to the node pool') -param nodeTaints array = [] - -@description('Any labels that should be applied to the node pool') -param nodeLabels object = {} - -@description('The subnet the node pool will use') -param subnetId string - -@description('The subnet the pods will use') -param podSubnetId string - -@description('OS Type for the node pool') -@allowed([ - 'Linux' - 'Windows' -]) -param osType string = 'Linux' - -resource aks 'Microsoft.ContainerService/managedClusters@2024-01-01' existing = { - name: AksName -} - -resource nodepool 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = { - parent: aks - name: PoolName - properties: { - mode: 'User' - vmSize: agentVMSize - count: agentCount - minCount: autoScale ? agentCount : null - maxCount: autoScale ? agentCountMax : null - enableAutoScaling: autoScale - availabilityZones: !empty(availabilityZones) ? availabilityZones : null - osDiskType: osDiskType - osDiskSizeGB: osDiskSizeGB - osType: osType - osSKU: osType == 'Linux' ? 'AzureLinux' : null - maxPods: maxPods - type: 'VirtualMachineScaleSets' - vnetSubnetID: !empty(subnetId) ? subnetId : null - podSubnetID: !empty(podSubnetId) ? podSubnetId : null - upgradeSettings: { - maxSurge: '33%' - } - nodeTaints: nodeTaints - nodeLabels: nodeLabels - } -} diff --git a/bicep/modules/aks_cluster.bicep b/bicep/modules/aks_cluster.bicep deleted file mode 100644 index a5d27031..00000000 --- a/bicep/modules/aks_cluster.bicep +++ /dev/null @@ -1,796 +0,0 @@ -/* - This is a custom module that configures Azure Kubernetes Service. - - ** Eventually this might move to Managed-Platform-Modules. ** -*/ - -targetScope = 'resourceGroup' - -/*______ ___ .______ ___ .___ ___. _______ .___________. _______ .______ _______. -| _ \ / \ | _ \ / \ | \/ | | ____|| || ____|| _ \ / | -| |_) | / ^ \ | |_) | / ^ \ | \ / | | |__ `---| |----`| |__ | |_) | | (----` -| ___/ / /_\ \ | / / /_\ \ | |\/| | | __| | | | __| | / \ \ -| | / _____ \ | |\ \----./ _____ \ | | | | | |____ | | | |____ | |\ \----.----) | -| _| /__/ \__\ | _| `._____/__/ \__\ |__| |__| |_______| |__| |_______|| _| `._____|_______/ -*/ - -//////////////////// -// Basic Details -//////////////////// -@minLength(1) -@maxLength(63) -@description('Used to name all resources') -param resourceName string - -@description('Specify the location of the AKS cluster.') -param location string = resourceGroup().location - -@description('Tags.') -param tags object = {} - -@description('The ID of the Azure AD tenant') -param aad_tenant_id string = '' - -@description('The ID of the Azure AD tenant') -param admin_ids array = [] - -@description('Use the paid sku for SLA rather than SLO') -param skuTierPaid bool = false - -@description('Kubernetes Version') -param aksVersion string = '1.28' - -@allowed([ - 'none' - 'patch' - 'stable' - 'rapid' - 'node-image' -]) -@description('AKS upgrade channel') -param upgradeChannel string = 'stable' - -@maxLength(80) -@description('The name of the NEW resource group to create the AKS cluster managed resources in') -param managedNodeResourceGroup string = '' - - -//////////////////// -// Compute Configuration -//////////////////// -@allowed([ - 'Burstable' - 'Standard' -]) -@description('The System Pool Preset sizing') -param clusterSize string = 'Burstable' - -@description('Override the default server type.') -param customVMSize string = '' - -@description('The System Pool Preset sizing') -param AutoscaleProfile object = { - 'balance-similar-node-groups': 'true' - expander: 'random' - 'max-empty-bulk-delete': '10' - 'max-graceful-termination-sec': '600' - 'max-node-provision-time': '15m' - 'max-total-unready-percentage': '45' - 'new-pod-scale-up-delay': '0s' - 'ok-total-unready-count': '3' - 'scale-down-delay-after-add': '10m' - 'scale-down-delay-after-delete': '20s' - 'scale-down-delay-after-failure': '3m' - 'scale-down-unneeded-time': '10m' - 'scale-down-unready-time': '20m' - 'scale-down-utilization-threshold': '0.5' - 'scan-interval': '10s' - 'skip-nodes-with-local-storage': 'true' - 'skip-nodes-with-system-pods': 'true' -} - -@description('The number of agents for the user node pool') -param agentCount int = 3 - -@description('The maximum number of nodes for the user node pool') -param agentCountMax int = 0 -var autoScale = agentCountMax > agentCount - -@minLength(3) -@maxLength(12) -@description('Name for user node pool') -param nodePoolName string = 'internal' - -@description('Only use the system node pool') -param JustUseSystemPool bool = false - -//////////////////// -// Required Items to link to other resources -//////////////////// - -@description('Specify the Log Analytics Workspace Id to use for monitoring.') -param workspaceId string - -@description('Specify the User Managed Identity Resource Id.') -param identityId string - - -//////////////////// -// Network Configuration -//////////////////// - -@allowed([ - 'azure' - 'kubenet' -]) -@description('The network plugin type') -param networkPlugin string = 'azure' - -@allowed([ - '' - 'Overlay' -]) -@description('The network plugin type') -param networkPluginMode string = 'Overlay' - -@allowed([ - '' - 'azure' - 'calico' - 'cilium' -]) -@description('The network policy to use.') -param networkPolicy string = 'cilium' - -@minLength(9) -@maxLength(18) -@description('The address range to use for pods') -param podCidr string = '192.168.0.0/16' - -@minLength(9) -@maxLength(18) -@description('The address range to use for services') -param serviceCidr string = '172.16.0.0/16' - -@minLength(7) -@maxLength(15) -@description('The IP address to reserve for DNS') -param dnsServiceIP string = '172.16.0.10' - -@allowed([ - 'loadBalancer' - 'natGateway' - 'userDefinedRouting' -]) -@description('Outbound traffic type for the egress traffic of your cluster') -param aksOutboundTrafficType string = 'loadBalancer' - -@description('DNS prefix. Defaults to {resourceName}-dns') -param dnsPrefix string = 'aks-${resourceGroup().name}' - -@minValue(1) -@maxValue(16) -@description('The effective outbound IP resources of the cluster NAT gateway') -param natGwIpCount int = 2 - -@minValue(4) -@maxValue(120) -@description('Outbound flow idle timeout in minutes for NatGw') -param natGwIdleTimeout int = 30 - -@description('Allocate pod ips dynamically') -param cniDynamicIpAllocation bool = false - - -//////////////////// -// BYO Network Configuration -//////////////////// - -@description('Are you providing a custom VNET') -param custom_vnet bool = false - -@description('Full resource id path of an existing subnet to use for AKS') -param aksSubnetId string = '' - -@description('Full resource id path of an existing pod subnet to use for AKS') -param aksPodSubnetId string = '' - - -//////////////////// -// Security Settings -//////////////////// - -@description('Enable private cluster') -param enablePrivateCluster bool = false - -@allowed([ - 'system' - 'none' - 'privateDnsZone' -]) -@description('Private cluster dns advertisment method, leverages the dnsApiPrivateZoneId parameter') -param privateClusterDnsMethod string = 'system' - -@description('The full Azure resource ID of the privatelink DNS zone to use for the AKS cluster API Server') -param dnsApiPrivateZoneId string = '' - -@description('The IP addresses that are allowed to access the API server') -param authorizedIPRanges array = [] - -@allowed([ - '' - 'audit' - 'deny' -]) -@description('Enable the Azure Policy addon') -param azurepolicy string = '' - -@allowed([ - 'Baseline' - 'Restricted' -]) -param azurePolicyInitiative string = 'Baseline' - - -//////////////////// -// Add Ons -//////////////////// - -@description('Enables Kubernetes Event-driven Autoscaling (KEDA)') -param kedaEnabled bool = false - -@description('Enables Open Service Mesh') -param openServiceMeshEnabled bool = false - -@description('Installs Azure Workload Identity into the cluster') -param workloadIdentityEnabled bool = false - -@description('Enable Microsoft Defender for Containers (preview)') -param defenderEnabled bool = false - -@description('Installs the AKS KV CSI provider') -param keyvaultEnabled bool = false - -@description('Rotation poll interval for the AKS KV CSI provider') -param keyVaultAksCSIPollInterval string = '2m' - -@description('Enable Azure AD integration on AKS') -param enable_aad bool = false - -@description('Enable RBAC using AAD') -param enableAzureRBAC bool = false - -@description('Enables SGX Confidential Compute plugin') -param sgxPlugin bool = false - -@description('Enables the Blob CSI driver') -param blobCSIDriver bool = false - -@description('Enables the File CSI driver') -param fileCSIDriver bool = true - -@description('Enables the Disk CSI driver') -param diskCSIDriver bool = true - -@description('Disable local K8S accounts for AAD enabled clusters') -param AksDisableLocalAccounts bool = false - -@description('Configures the cluster as an OIDC issuer for use with Workload Identity') -param oidcIssuer bool = false - -@description('Enable Web App Routing') -param warIngressNginx bool = false - -@description('Specifies whether to enable ImageCleaner on AKS cluster. The default value is false.') -param enableImageCleaner bool = false - -@description('Specifies whether ImageCleaner scanning interval in hours.') -param imageCleanerIntervalHours int = 24 - -// Preview feature requires: az feature register --namespace "Microsoft.ContainerService" --name "AzureServiceMeshPreview" -@allowed(['', 'Istio']) -@description('The service mesh profile to use') -param serviceMeshProfile string = '' - -@allowed([ - '' - 'External' - 'Internal' - 'Both' -]) -@description('The ingress gateway to use for the Istio service mesh') -param istioIngressGatewayMode string = '' -param istioRevision string = 'asm-1-18' - - -/*__ ____ ___ .______ __ ___ .______ __ _______ _______. -\ \ / / / \ | _ \ | | / \ | _ \ | | | ____| / | - \ \/ / / ^ \ | |_) | | | / ^ \ | |_) | | | | |__ | (----` - \ / / /_\ \ | / | | / /_\ \ | _ < | | | __| \ \ - \ / / _____ \ | |\ \----.| | / _____ \ | |_) | | `----.| |____.----) | - \__/ /__/ \__\ | _| `._____||__| /__/ \__\ |______/ |_______||_______|_______/ -*/ - -@description('The name of the AKS cluster.') -var name = '${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}' - -var serviceMeshProfileObj = { - istio: { - components: { - ingressGateways: empty(istioIngressGatewayMode) ? null : union( - istioIngressGatewayMode == 'Internal' ? array(ingressModes.internal) : [], - istioIngressGatewayMode == 'External' ? array(ingressModes.external) : [], - istioIngressGatewayMode == 'Both' ? array(ingressModes.internal) : [], - istioIngressGatewayMode == 'Both' ? array(ingressModes.external) : [] - ) - } - revisions: [ - istioRevision - ] - } - mode: 'Istio' -} - -@description('This resolves the friendly natGateway to the actual outbound traffic type value used by AKS') -var outboundTrafficType = aksOutboundTrafficType=='natGateway' ? ( custom_vnet ? 'userAssignedNATGateway' : 'managedNATGateway' ) : aksOutboundTrafficType - -@description('System Pool presets are derived from the recommended system pool specs') -var systemPoolPresets = { - Burstable : { - vmSize: 'Standard_B2ms' - minCount: 1 - maxCount: 3 - availabilityZones: [ - '1' - '2' - '3' - ] - osDiskType: 'Managed' - osDiskSize: 128 - maxPods: 30 - } - Standard : { - vmSize: 'Standard_D2s_v5' - minCount: 1 - maxCount: 3 - availabilityZones: [ - '1' - '2' - '3' - ] - osDiskType: 'Managed' - osDiskSize: 128 - maxPods: 30 - } -} - -var systemPoolProfile = { - name: 'default' - mode: 'System' - osType: 'Linux' - osSKU: 'AzureLinux' - type: 'VirtualMachineScaleSets' - osDiskType: systemPoolPresets[clusterSize].osDiskType - osDiskSizeGB: systemPoolPresets[clusterSize].osDiskSize - vmSize: empty(customVMSize) ? systemPoolPresets[clusterSize].vmSize : customVMSize - count: systemPoolPresets[clusterSize].minCount - minCount: systemPoolPresets[clusterSize].minCount - maxCount: systemPoolPresets[clusterSize].maxCount - availabilityZones: systemPoolPresets[clusterSize].availabilityZones - enableAutoScaling: true - maxPods: systemPoolPresets[clusterSize].maxPods - vnetSubnetID: !empty(aksSubnetId) ? aksSubnetId : null - podSubnetID: !empty(aksPodSubnetId) ? aksPodSubnetId : null - upgradeSettings: { - maxSurge: '33%' - } - nodeTaints: [ - JustUseSystemPool ? '' : 'CriticalAddonsOnly=true:NoSchedule' - ] -} - -@description('First User Pool presets') -var userPoolPresets = { - Burstable : { - vmSize: 'Standard_B4ms' - minCount: 5 - maxCount: 20 - availabilityZones: [ - '1' - '2' - '3' - ] - osDiskType: 'Managed' - osDiskSize: 128 - maxPods: 60 - } - Standard : { - vmSize: 'Standard_D4s_v5' - minCount: 5 - maxCount: 20 - availabilityZones: [ - '1' - '2' - '3' - ] - osDiskType: 'Managed' - osDiskSize: 128 - maxPods: 60 - } -} - -var userPoolProfile = { - name: nodePoolName - mode: 'User' - osType: 'Linux' - osSKU: 'AzureLinux' - type: 'VirtualMachineScaleSets' - osDiskType: userPoolPresets[clusterSize].osDiskType - osDiskSizeGB: userPoolPresets[clusterSize].osDiskSize - vmSize: empty(customVMSize) ? userPoolPresets[clusterSize].vmSize : customVMSize - count: userPoolPresets[clusterSize].minCount - minCount: userPoolPresets[clusterSize].minCount - maxCount: userPoolPresets[clusterSize].maxCount - availabilityZones: userPoolPresets[clusterSize].availabilityZones - enableAutoScaling: true - maxPods: userPoolPresets[clusterSize].maxPods - vnetSubnetID: !empty(aksSubnetId) ? aksSubnetId : null - podSubnetID: !empty(aksPodSubnetId) ? aksPodSubnetId : null - upgradeSettings: { - maxSurge: '33%' - } -} - -var agentPoolProfiles = JustUseSystemPool ? array(systemPoolProfile) : concat(array(systemPoolProfile), array(userPoolProfile)) - - -var akssku = skuTierPaid ? 'Standard' : 'Free' - -var aks_addons = union({ - azurepolicy: { - config: { - version: !empty(azurepolicy) ? 'v2' : null - } - enabled: !empty(azurepolicy) - } - azureKeyvaultSecretsProvider: { - config: { - enableSecretRotation: 'true' - rotationPollInterval: keyVaultAksCSIPollInterval - } - enabled: keyvaultEnabled - } - openServiceMesh: { - enabled: openServiceMeshEnabled - config: {} - } - ACCSGXDevicePlugin: { - enabled: sgxPlugin - config: {} - } -}, !(empty(workspaceId)) ? { - omsagent: { - enabled: !(empty(workspaceId)) - config: { - logAnalyticsWorkspaceResourceID: !(empty(workspaceId)) ? workspaceId : null - } - }} : {}) - - -@description('Sets the private dns zone id if provided') -var aksPrivateDnsZone = privateClusterDnsMethod=='privateDnsZone' ? (!empty(dnsApiPrivateZoneId) ? dnsApiPrivateZoneId : 'system') : privateClusterDnsMethod - -@description('Needing to separately declare and union this because of https://github.com/Azure/AKS-Construction/issues/344') -var managedNATGatewayProfile = { - natGatewayProfile : { - managedOutboundIPProfile: { - count: natGwIpCount - } - idleTimeoutInMinutes: natGwIdleTimeout - } -} - -@description('Needing to separately declare and union this because of https://github.com/Azure/AKS/issues/2774') -var azureDefenderSecurityProfile = { - securityProfile : { - defender: { - logAnalyticsWorkspaceResourceId: workspaceId - securityMonitoring: { - enabled: defenderEnabled - } - } - } -} - - -var aksProperties = union({ - kubernetesVersion: aksVersion - enableRBAC: true - dnsPrefix: dnsPrefix - aadProfile: enable_aad ? { - managed: true - enableAzureRBAC: enableAzureRBAC - tenantID: aad_tenant_id - adminGroupObjectIDs: empty(admin_ids) ? null : admin_ids - } : null - apiServerAccessProfile: !empty(authorizedIPRanges) ? { - authorizedIPRanges: authorizedIPRanges - } : { - enablePrivateCluster: enablePrivateCluster - privateDNSZone: enablePrivateCluster ? aksPrivateDnsZone : '' - enablePrivateClusterPublicFQDN: enablePrivateCluster && privateClusterDnsMethod=='none' - } - agentPoolProfiles: agentPoolProfiles - workloadAutoScalerProfile: { - keda: { - enabled: kedaEnabled - } - } - networkProfile: { - loadBalancerSku: 'standard' - networkPlugin: networkPlugin - #disable-next-line BCP036 //Disabling validation of this parameter to cope with empty string to indicate no Network Policy required. - networkPolicy: networkPolicy - networkPluginMode: networkPlugin=='azure' ? networkPluginMode : '' - networkDataplane: networkPolicy=='cilium' ? networkPolicy : '' - podCidr: networkPlugin=='kubenet' || networkPluginMode=='Overlay' || cniDynamicIpAllocation ? podCidr : null - serviceCidr: serviceCidr - dnsServiceIP: dnsServiceIP - outboundType: outboundTrafficType - } - disableLocalAccounts: AksDisableLocalAccounts && enable_aad - autoUpgradeProfile: {upgradeChannel: upgradeChannel} - addonProfiles: aks_addons - autoScalerProfile: autoScale ? AutoscaleProfile : {} - oidcIssuerProfile: { - enabled: oidcIssuer - } - securityProfile: { - workloadIdentity: { - enabled: workloadIdentityEnabled - } - defender: { - logAnalyticsWorkspaceResourceId: defenderEnabled ? workspaceId : null - securityMonitoring: { - enabled: defenderEnabled - } - } - imageCleaner: { - enabled: enableImageCleaner - intervalHours: imageCleanerIntervalHours - } - } - ingressProfile: { - webAppRouting: { - enabled: warIngressNginx - } - } - storageProfile: { - blobCSIDriver: { - enabled: blobCSIDriver - } - diskCSIDriver: { - enabled: diskCSIDriver - } - fileCSIDriver: { - enabled: fileCSIDriver - } - } -}, -outboundTrafficType == 'managedNATGateway' ? managedNATGatewayProfile : {}, -defenderEnabled ? azureDefenderSecurityProfile : {}, -!empty(managedNodeResourceGroup) ? { nodeResourceGroup: managedNodeResourceGroup} : {}, -!empty(serviceMeshProfile) ? { serviceMeshProfile: serviceMeshProfileObj } : {} -) - -var ingressModes = { - external: { - enabled: true - mode: 'External' - } - internal: { - enabled: true - mode: 'Internal' - } -} - - -/* -.______ _______ _______. ______ __ __ .______ ______ _______ _______. -| _ \ | ____| / | / __ \ | | | | | _ \ / || ____| / | -| |_) | | |__ | (----`| | | | | | | | | |_) | | ,----'| |__ | (----` -| / | __| \ \ | | | | | | | | | / | | | __| \ \ -| |\ \----.| |____.----) | | `--' | | `--' | | |\ \----.| `----.| |____.----) | -| _| `._____||_______|_______/ \______/ \______/ | _| `._____| \______||_______|_______/ -*/ - -resource aks 'Microsoft.ContainerService/managedClusters@2024-02-01' = { - name: length(name) > 63 ? substring(name, 0, 63) : name - location: location - tags: tags - - identity: { - type: 'UserAssigned' - userAssignedIdentities: { - '${identityId}': {} - } - } - - - properties: aksProperties - - sku: { - name: 'Base' - tier: akssku - } -} - -/* - ______ __ __ .___________..______ __ __ .___________. - / __ \ | | | | | || _ \ | | | | | | -| | | | | | | | `---| |----`| |_) | | | | | `---| |----` -| | | | | | | | | | | ___/ | | | | | | -| `--' | | `--' | | | | | | `--' | | | - \______/ \______/ |__| | _| \______/ |__| -*/ -output aksClusterName string = aks.name -output aksOidcIssuerUrl string = oidcIssuer ? aks.properties.oidcIssuerProfile.issuerURL : '' -output userNodePoolName string = nodePoolName -output systemNodePoolName string = JustUseSystemPool ? nodePoolName : 'npsystem' - -output aksPrivateDnsZone string = aksPrivateDnsZone -output privateFQDN string = enablePrivateCluster && privateClusterDnsMethod != 'none' ? aks.properties.privateFQDN : '' -// Dropping cluster name at start of privateFQDN to get private dns zone name. -output aksPrivateDnsZoneName string = enablePrivateCluster && privateClusterDnsMethod != 'none' ? join(skip(split(aks.properties.privateFQDN, '.'),1),'.') : '' - - - -@description('This output can be directly leveraged when creating a ManagedId Federated Identity') -output aksOidcFedIdentityProperties object = { - issuer: oidcIssuer ? aks.properties.oidcIssuerProfile.issuerURL : '' - audiences: ['api://AzureADTokenExchange'] - subject: 'system:serviceaccount:ns:svcaccount' -} - -@description('The name of the managed resource group AKS uses') -output aksNodeResourceGroup string = aks.properties.nodeResourceGroup - -@description('The Azure resource id for the AKS cluster') -output aksResourceId string = aks.id - - - -//////////////// -// Policy -//////////////// - -var policySetBaseline = '/providers/Microsoft.Authorization/policySetDefinitions/a8640138-9b0a-4a28-b8cb-1666c838647d' -var policySetRestrictive = '/providers/Microsoft.Authorization/policySetDefinitions/42b8ef37-b724-4e24-bbc8-7a7708edfe00' - -resource aks_policies 'Microsoft.Authorization/policyAssignments@2023-04-01' = if (!empty(azurepolicy)) { - name: '${aks.name}-${azurePolicyInitiative}' - location: location - properties: { - policyDefinitionId: azurePolicyInitiative == 'Baseline' ? policySetBaseline : policySetRestrictive - parameters: { - excludedNamespaces: { - value: [ - 'kube-system' - 'gatekeeper-system' - 'azure-arc' - 'cluster-baseline-setting' - ] - } - effect: { - value: azurepolicy - } - } - metadata: { - assignedBy: 'OSDU Developer' - } - displayName: 'Kubernetes cluster pod security ${azurePolicyInitiative} standards for Linux-based workloads' - description: 'As per: https://github.com/Azure/azure-policy/blob/master/built-in-policies/policySetDefinitions/Kubernetes/' - } -} - -@description('If automated deployment, for the 3 automated user assignments, set Principal Type on each to "ServicePrincipal" rarter than "User"') -param automatedDeployment bool = false - -@description('The principal ID to assign the AKS admin role.') -param adminPrincipalId string = '' -// for AAD Integrated Cluster wusing 'enableAzureRBAC', add Cluster admin to the current user! -var buildInAKSRBACClusterAdmin = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b') -resource aks_admin_role_assignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (enableAzureRBAC && !empty(adminPrincipalId)) { - scope: aks // Use when specifying a scope that is different than the deployment scope - name: guid(aks.id, 'aksadmin', buildInAKSRBACClusterAdmin) - properties: { - roleDefinitionId: buildInAKSRBACClusterAdmin - principalType: automatedDeployment ? 'ServicePrincipal' : 'User' - principalId: adminPrincipalId - } -} - -// Haven't figured out how to use this properly yet. -// resource appConfigExtension 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = { -// name: 'appconfigurationkubernetesprovider' -// scope: aks -// properties: { -// autoUpgradeMinorVersion: true -// configurationSettings: { -// 'global.clusterType': 'managedclusters' -// autoscaling.enabled: true -// } -// extensionType: 'microsoft.appconfiguration' -// releaseTrain: 'preview' -// } -// } - - -/* - _______ __ __ __ ___ ___ -| ____|| | | | | | \ \ / / -| |__ | | | | | | \ V / -| __| | | | | | | > < -| | | `----.| `--' | / . \ -|__| |_______| \______/ /__/ \__\ -*/ -param fluxGitOpsAddon bool = false - -resource fluxAddon 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = if(fluxGitOpsAddon) { - name: 'flux' - scope: aks - properties: { - extensionType: 'microsoft.flux' - autoUpgradeMinorVersion: true - // https://learn.microsoft.com/en-us/azure/azure-arc/kubernetes/conceptual-gitops-flux2#opt-out-of-multi-tenancy. - configurationSettings: { - 'multiTenancy.enforce': 'false' - } - releaseTrain: 'Stable' - scope: { - cluster: { - releaseNamespace: 'flux-system' - } - } - configurationProtectedSettings: {} - } - dependsOn: [daprExtension] //Chaining dependencies because of: https://github.com/Azure/AKS-Construction/issues/385 -} -output fluxReleaseNamespace string = fluxGitOpsAddon ? fluxAddon.properties.scope.cluster.releaseNamespace : '' - - -/* - _______ ___ .______ .______ -| \ / \ | _ \ | _ \ -| .--. | / ^ \ | |_) | | |_) | -| | | | / /_\ \ | ___/ | / -| '--' | / _____ \ | | | |\ \----. -|_______/ /__/ \__\ | _| | _| `._____| -*/ - - -@description('Add the Dapr extension') -param daprAddon bool = false -@description('Enable high availability (HA) mode for the Dapr control plane') -param daprAddonHA bool = false - -resource daprExtension 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = if(daprAddon) { - name: 'dapr' - scope: aks - properties: { - extensionType: 'Microsoft.Dapr' - autoUpgradeMinorVersion: true - releaseTrain: 'Stable' - configurationSettings: { - 'global.ha.enabled': '${daprAddonHA}' - } - scope: { - cluster: { - releaseNamespace: 'dapr-system' - } - } - configurationProtectedSettings: {} - } -} - -output daprReleaseNamespace string = daprAddon ? daprExtension.properties.scope.cluster.releaseNamespace : '' diff --git a/bicep/modules/aks_grafana.bicep b/bicep/modules/aks_grafana.bicep deleted file mode 100644 index c1e7ef52..00000000 --- a/bicep/modules/aks_grafana.bicep +++ /dev/null @@ -1,137 +0,0 @@ -// Parameters -@description('Specifies the name of the Azure Monitor managed service for Grafana resource.') -param name string - -@description('Specifies the location of the Azure Managed Grafana resource.') -param location string = resourceGroup().location - -@description('Specifies the name of the Azure Monitor managed service for Prometheus resource.') -param prometheusName string - -@description('Specifies the sku of the Azure Managed Grafana resource.') -param skuName string = 'Standard' - -@description('Specifies the api key setting of the Azure Managed Grafana resource.') -@allowed([ - 'Disabled' - 'Enabled' -]) -param apiKey string = 'Enabled' - -@description('Specifies the scope for dns deterministic name hash calculation.') -@allowed([ - 'TenantReuse' -]) -param autoGeneratedDomainNameLabelScope string = 'TenantReuse' - -@description('Specifies whether the Azure Managed Grafana resource uses deterministic outbound IPs.') -@allowed([ - 'Disabled' - 'Enabled' -]) -param deterministicOutboundIP string = 'Disabled' - -@description('Specifies the the state for enable or disable traffic over the public interface for the the Azure Managed Grafana resource.') -@allowed([ - 'Disabled' - 'Enabled' -]) -param publicNetworkAccess string = 'Enabled' - -@description('The zone redundancy setting of the Azure Managed Grafana resource.') -@allowed([ - 'Disabled' - 'Enabled' -]) -param zoneRedundancy string = 'Disabled' - -@description('Specifies the object id of an Azure Active Directory user. In general, this the object id of the system administrator who deploys the Azure resources.') -param userId string = '' - -@description('Specifies the resource tags for the Azure Monitor managed service for Prometheus resource.') -param tags object - - - -// Resources -resource mmonitoringReaderRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { - name: '43d0d8ad-25c7-4714-9337-8ba259a9fe05' - scope: subscription() -} - -resource monitoringDataReaderRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { - name: 'b0d8363b-8ddd-447d-831f-62ca05bff136' - scope: subscription() -} - -resource grafanaAdminRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { - name: '22926164-76b3-42b3-bc55-97df8dab3e41' - scope: subscription() -} - -resource managedPrometheus 'Microsoft.Monitor/accounts@2023-04-03' existing = { - name: prometheusName -} - -resource managedGrafana 'Microsoft.Dashboard/grafana@2022-08-01' = { - name: name - location: location - tags: tags - sku: { - name: skuName - } - identity: { - type: 'SystemAssigned' - } - properties: { - apiKey: apiKey - autoGeneratedDomainNameLabelScope: autoGeneratedDomainNameLabelScope - deterministicOutboundIP: deterministicOutboundIP - grafanaIntegrations: { - azureMonitorWorkspaceIntegrations: [{ - azureMonitorWorkspaceResourceId: managedPrometheus.id - }] - } - publicNetworkAccess: publicNetworkAccess - zoneRedundancy: zoneRedundancy - } -} - -// Assign the Monitoring Reader role to the Azure Managed Grafana system-assigned managed identity at the workspace scope -resource monitoringReaderRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(name, managedPrometheus.name, mmonitoringReaderRole.id) - scope: managedPrometheus - properties: { - roleDefinitionId: mmonitoringReaderRole.id - principalId: managedGrafana.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Assign the Monitoring Data Reader role to the Azure Managed Grafana system-assigned managed identity at the workspace scope -resource monitoringDataReaderRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(name, managedPrometheus.name, monitoringDataReaderRole.id) - scope: managedPrometheus - properties: { - roleDefinitionId: monitoringDataReaderRole.id - principalId: managedGrafana.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Assign the Grafana Admin role to the Microsoft Entra ID user at the Azure Managed Grafana resource scope -resource grafanaAdminRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(userId)) { - name: guid(name, userId, grafanaAdminRole.id) - scope: managedGrafana - properties: { - roleDefinitionId: grafanaAdminRole.id - principalId: userId - principalType: 'User' - } -} - -// Outputs -output id string = managedGrafana.id -output name string = managedGrafana.name -output location string = managedGrafana.location -output principalId string = managedGrafana.identity.principalId diff --git a/bicep/modules/aks_identity.bicep b/bicep/modules/aks_identity.bicep deleted file mode 100644 index 1f41bf22..00000000 --- a/bicep/modules/aks_identity.bicep +++ /dev/null @@ -1,28 +0,0 @@ -// Parameters -@description('Specifies the name of the existing AKS cluster.') -param aksClusterName string - -@description('Specifies the name of the existing container registry.') -param acrName string - -// Variables -var acrPullRoleDefinitionId = resourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') - -// Resources -resource aksCluster 'Microsoft.ContainerService/managedClusters@2023-10-02-preview' existing = { - name: aksClusterName -} - -resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-11-01-preview' existing = { - name: acrName -} - -resource acrPullRoleAssignmentName 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(aksCluster.name, containerRegistry.id, acrPullRoleDefinitionId) - scope: containerRegistry - properties: { - roleDefinitionId: acrPullRoleDefinitionId - principalId: any(aksCluster.properties.identityProfile.kubeletidentity).objectId - principalType: 'ServicePrincipal' - } -} diff --git a/bicep/modules/aks_prometheus.bicep b/bicep/modules/aks_prometheus.bicep deleted file mode 100644 index 0569b787..00000000 --- a/bicep/modules/aks_prometheus.bicep +++ /dev/null @@ -1,486 +0,0 @@ -// Parameters - -@description('Specifies the name of the Azure Monitor managed service for Prometheus resource.') -param name string - -@description('Specifies the location of the Azure Monitor managed service for Prometheus resource.') -param location string = resourceGroup().location - -@description('Specifies the name of the AKS cluster.') -param clusterName string - -@description('Specifies whether or not public endpoint access is allowed for the Azure Monitor managed service for Prometheus resource.') -@allowed([ - 'Enabled' - 'Disabled' -]) -param publicNetworkAccess string = 'Enabled' - -@description('Specifies the resource id of an Action Group resource. If empty, no action is specifies for metric alerts.') -param actionGroupId string = '' - -@description('Specifies the resource tags for the Azure Monitor managed service for Prometheus resource.') -param tags object - -var nodeRecordingRuleGroupPrefix = 'NodeRecordingRulesRuleGroup-' -var nodeRecordingRuleGroupName = '${nodeRecordingRuleGroupPrefix}${clusterName}' -var nodeRecordingRuleGroupDescription = 'Node Recording Rules RuleGroup' -var version = ' - 0.1' - -// Resources -resource aksCluster 'Microsoft.ContainerService/managedClusters@2023-06-02-preview' existing = { - name: clusterName -} - -resource azureMonitorWorkspace 'Microsoft.Monitor/accounts@2023-04-03' = { - name: name - location: location - tags: tags -} - -resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2022-06-01' = { - name: 'MSProm-${location}-${clusterName}' - location: location - kind: 'Linux' - tags: tags - properties: { - networkAcls: { - publicNetworkAccess: publicNetworkAccess - } - } -} - -resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2022-06-01' = { - name: 'MSProm-${location}-${clusterName}' - location: location - tags: tags - - properties: { - dataCollectionEndpointId: dataCollectionEndpoint.id - dataSources: { - prometheusForwarder: [ - { - name: 'PrometheusDataSource' - streams: [ - 'Microsoft-PrometheusMetrics' - ] - labelIncludeFilter: {} - } - ] - } - destinations: { - monitoringAccounts: [ - { - accountResourceId: azureMonitorWorkspace.id - name: 'MonitoringAccount1' - } - ] - } - dataFlows: [ - { - streams: [ - 'Microsoft-PrometheusMetrics' - ] - destinations: [ - 'MonitoringAccount1' - ] - } - ] - } -} - -resource dataCollectionRuleAssociation 'Microsoft.Insights/dataCollectionRuleAssociations@2022-06-01' = { - name: 'MSProm-${location}-${clusterName}' - scope: aksCluster - properties: { - dataCollectionRuleId: dataCollectionRule.id - description: 'Association of data collection rule. Deleting this association will break the data collection for this AKS Cluster.' - } -} - -resource nodeRecordingRuleGroup 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: nodeRecordingRuleGroupName - location: location - properties: { - description: '${nodeRecordingRuleGroupDescription}${version}' - scopes: [ - azureMonitorWorkspace.id - ] - enabled: true - clusterName: clusterName - interval: 'PT1M' - rules: [ - { - record: 'instance:node_num_cpu:sum' - expression: 'count without (cpu, mode) ( node_cpu_seconds_total{job="node",mode="idle"})' - } - { - record: 'instance:node_cpu_utilisation:rate5m' - expression: '1 - avg without (cpu) ( sum without (mode) (rate(node_cpu_seconds_total{job="node", mode=~"idle|iowait|steal"}[5m])))' - } - { - record: 'instance:node_load1_per_cpu:ratio' - expression: '( node_load1{job="node"}/ instance:node_num_cpu:sum{job="node"})' - } - { - record: 'instance:node_memory_utilisation:ratio' - expression: '1 - ( ( node_memory_MemAvailable_bytes{job="node"} or ( node_memory_Buffers_bytes{job="node"} + node_memory_Cached_bytes{job="node"} + node_memory_MemFree_bytes{job="node"} + node_memory_Slab_bytes{job="node"} ) )/ node_memory_MemTotal_bytes{job="node"})' - } - { - record: 'instance:node_vmstat_pgmajfault:rate5m' - expression: 'rate(node_vmstat_pgmajfault{job="node"}[5m])' - } - { - record: 'instance_device:node_disk_io_time_seconds:rate5m' - expression: 'rate(node_disk_io_time_seconds_total{job="node", device!=""}[5m])' - } - { - record: 'instance_device:node_disk_io_time_weighted_seconds:rate5m' - expression: 'rate(node_disk_io_time_weighted_seconds_total{job="node", device!=""}[5m])' - } - { - record: 'instance:node_network_receive_bytes_excluding_lo:rate5m' - expression: 'sum without (device) ( rate(node_network_receive_bytes_total{job="node", device!="lo"}[5m]))' - } - { - record: 'instance:node_network_transmit_bytes_excluding_lo:rate5m' - expression: 'sum without (device) ( rate(node_network_transmit_bytes_total{job="node", device!="lo"}[5m]))' - } - { - record: 'instance:node_network_receive_drop_excluding_lo:rate5m' - expression: 'sum without (device) ( rate(node_network_receive_drop_total{job="node", device!="lo"}[5m]))' - } - { - record: 'instance:node_network_transmit_drop_excluding_lo:rate5m' - expression: 'sum without (device) ( rate(node_network_transmit_drop_total{job="node", device!="lo"}[5m]))' - } - ] - } -} - -resource communityALerts 'Microsoft.AlertsManagement/prometheusRuleGroups@2021-07-22-preview' = { - name: 'CommunityCIAlerts-${clusterName}' - location: location - properties: { - description: 'Kubernetes Alert RuleGroup-communityCIAlerts - 0.1' - scopes: [ - azureMonitorWorkspace.id - ] - clusterName: clusterName - enabled: true - interval: 'PT1M' - rules: [ - { - alert: 'KubePodCrashLooping' - expression: 'max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics"}[5m]) >= 1' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubePodNotReady' - expression: 'sum by (namespace, pod, cluster) ( max by(namespace, pod, cluster) ( kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"} ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) ( 1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) )) > 0' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeDeploymentReplicasMismatch' - expression: '( kube_deployment_spec_replicas{job="kube-state-metrics"} > kube_deployment_status_replicas_available{job="kube-state-metrics"}) and ( changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m]) == 0)' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeStatefulSetReplicasMismatch' - expression: '( kube_statefulset_status_replicas_ready{job="kube-state-metrics"} != kube_statefulset_status_replicas{job="kube-state-metrics"}) and ( changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m]) == 0)' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeJobNotCompleted' - expression: 'time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics"} and kube_job_status_active{job="kube-state-metrics"} > 0) > 43200' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeJobFailed' - expression: 'kube_job_failed{job="kube-state-metrics"} > 0' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeHpaReplicasMismatch' - expression: '(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"} !=kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}) and(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} >kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"}) and(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} 1.5' - for: 'PT5M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeMemoryQuotaOvercommit' - expression: 'sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) /sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) > 1.5' - for: 'PT5M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeQuotaAlmostFull' - expression: 'kube_resourcequota{job="kube-state-metrics", type="used"} / ignoring(instance, job, type)(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) > 0.9 < 1' - for: 'PT15M' - labels: { - severity: 'info' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeVersionMismatch' - expression: 'count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeNodeNotReady' - expression: 'kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeNodeUnreachable' - expression: '(kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeletTooManyPods' - expression: 'count by(cluster, node) ( (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"}))/max by(cluster, node) ( kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1) > 0.95' - for: 'PT15M' - labels: { - severity: 'info' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - { - alert: 'KubeNodeReadinessFlapping' - expression: 'sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (cluster, node) > 2' - for: 'PT15M' - labels: { - severity: 'warning' - } - severity: 3 - enabled: true - resolveConfiguration: { - autoResolved: true - timeToResolve: 'PT10M' - } - actions: actionGroupId == '' ? [] : [ - { - actionGroupId: actionGroupId - } - ] - } - ] - } -} - -// Outputs -output id string = azureMonitorWorkspace.id -output name string = azureMonitorWorkspace.name -output location string = azureMonitorWorkspace.location -output accountId string = azureMonitorWorkspace.properties.accountId -output prometheusQueryEndpoint string = azureMonitorWorkspace.properties.metrics.prometheusQueryEndpoint -output internalId string = azureMonitorWorkspace.properties.metrics.internalId diff --git a/bicep/modules/blade_common.bicep b/bicep/modules/blade_common.bicep index 519748db..050895f5 100644 --- a/bicep/modules/blade_common.bicep +++ b/bicep/modules/blade_common.bicep @@ -34,6 +34,9 @@ param workspaceResourceId string @description('The Diagnostics Workspace Name') param workspaceName string +@description('Conditional. The name of the parent user assigned identity. Required if the template is used in a standalone deployment.') +param userAssignedIdentityName string + @description('The managed identity name for deployment scripts') param deploymentScriptIdentity string @@ -68,6 +71,7 @@ var commonLayerConfig = { 'system' 'azure-webjobs-hosts' 'azure-webjobs-eventhub' + 'gitops' ] tables: [ 'partitionInfo' @@ -93,6 +97,10 @@ var commonLayerConfig = { } } +resource userAssignedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { + name: userAssignedIdentityName +} + module insights 'br/public:avm/res/insights/component:0.3.0' = { name: '${bladeConfig.sectionName}-insights' params: { @@ -239,42 +247,6 @@ module keyvaultSecrets './keyvault_secrets.bicep' = { } } -// Deployment Scripts are not enabled yet for Private Link -// https://github.com/Azure/bicep/issues/6540 -module sshKey './script-sshkeypair/main.bicep' = { - name: '${bladeConfig.sectionName}-keyvault-sshkey' - params: { - kvName: keyvault.outputs.name - location: location - - useExistingManagedIdentity: true - managedIdentityName: deploymentScriptIdentity - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName:resourceGroup().name - - sshKeyName: 'PrivateLinkSSHKey-' - } -} - -module certificates './script-kv-certificate/main.bicep' = { - name: '${bladeConfig.sectionName}-keyvault-cert' - params: { - kvName: keyvault.outputs.name - location: location - - useExistingManagedIdentity: true - managedIdentityName: deploymentScriptIdentity - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName: resourceGroup().name - - certificateNames: [ - 'https-certificate' - ] - initialScriptDelay: '0' - validity: 24 - } -} - resource vaultDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (enablePrivateLink) { name: 'privatelink.vaultcore.azure.net' location: 'global' @@ -352,6 +324,35 @@ module configStorage './storage-account/main.bicep' = { } } +// var directoryUploads = [ +// { +// directory: 'software' +// } +// { +// directory: 'charts' +// } +// { +// directory: 'stamp' +// } +// ] + +// @batchSize(1) +// module gitOpsUpload './software-upload/main.bicep' = [for item in directoryUploads: { +// name: '${bladeConfig.sectionName}-storage-${item.directory}-upload' +// params: { +// storageAccountName: configStorage.outputs.name +// location: location +// useExistingManagedIdentity: true +// managedIdentityName: userAssignedIdentity.name +// existingManagedIdentitySubId: subscription().subscriptionId +// existingManagedIdentityResourceGroupName: resourceGroup().name +// directoryName: item.directory +// } +// dependsOn: [ +// configStorage +// ] +// }] + resource storageDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (enablePrivateLink) { name: storageDnsZoneName location: 'global' diff --git a/bicep/modules/blade_manage.bicep b/bicep/modules/blade_manage.bicep deleted file mode 100644 index f3ebcd3d..00000000 --- a/bicep/modules/blade_manage.bicep +++ /dev/null @@ -1,139 +0,0 @@ -//*****************************************************************// -// Manage Section // -//*****************************************************************// - -type bladeSettings = { - @description('The name of the section name') - sectionName: string - @description('The display name of the section') - displayName: string -} - -type manageSettings = { - @description('The settings for the virtual machine') - machine: machineSettings - @description('The settings for the bastion') - bastion: bastionSettings -} - -type machineSettings = { - @description('The size of the virtual machine') - vmSize: string - @description('The publisher of the image') - imagePublisher: string - @description('The offer of the image') - imageOffer: string - @description('The SKU of the image') - imageSku: string - @description('The authentication type of the virtual machine') - authenticationType: string -} - -type bastionSkuType = 'Basic' | 'Standard' - -type bastionSettings = { - @description('The name of the SKU') - skuName: bastionSkuType -} - - -@description('The configuration for the blade section.') -param bladeConfig bladeSettings - -@description('The location of resources to deploy') -param location string - -@description('The tags to apply to the resources') -param tags object = {} - -@description('Feature Flag to Enable Telemetry') -param enableTelemetry bool = false - - -@description('Specifies the name of the administrator account of the virtual machine.') -param vmAdminUsername string - -@description('Specifies the SSH Key or password for the virtual machine. SSH key is recommended.') -@secure() -param vmAdminPasswordOrKey string = '' - -@description('Feature Flag to Enable Bastion') -param enableBastion bool - -@description('The name of the Key Vault where the secret exists') -param kvName string - -@description('The Id of the virtual network') -param vnetId string - -@description('The Id of the subnet for the virtual machine') -param vmSubnetId string - -@description('The workspace name for diagnostics') -param workspaceName string - -@description('The configuration for the manage section.') -param manageLayerConfig manageSettings - - -/*.______ ___ _______.___________. __ ______ .__ __. -| _ \ / \ / | || | / __ \ | \ | | -| |_) | / ^ \ | (----`---| |----`| | | | | | | \| | -| _ < / /_\ \ \ \ | | | | | | | | | . ` | -| |_) | / _____ \ .----) | | | | | | `--' | | |\ | -|______/ /__/ \__\ |_______/ |__| |__| \______/ |__| \__| -*/ - -module bastionHost 'br/public:avm/res/network/bastion-host:0.2.1' = if (enableBastion) { - name: '${bladeConfig.sectionName}-bastion' - params: { - name: 'bh-${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' - skuName: manageLayerConfig.bastion.skuName - virtualNetworkResourceId: vnetId - location: location - enableTelemetry: enableTelemetry - - tags: union( - tags, - { - layer: bladeConfig.displayName - } - ) - } -} - - -/* -.___ ___. ___ ______ __ __ __ .__ __. _______ -| \/ | / \ / || | | | | | | \ | | | ____| -| \ / | / ^ \ | ,----'| |__| | | | | \| | | |__ -| |\/| | / /_\ \ | | | __ | | | | . ` | | __| -| | | | / _____ \ | `----.| | | | | | | |\ | | |____ -|__| |__| /__/ \__\ \______||__| |__| |__| |__| \__| |_______| -*/ - -resource existingVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = if (enableBastion) { - name: kvName -} - -module virtualMachine './virtual_machine.bicep' = if (enableBastion) { - name: '${bladeConfig.sectionName}-machine' - params: { - vmName: 'vm-${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' - vmSize: manageLayerConfig.machine.vmSize - - // Assign Tags - tags: union( - tags, - { - layer: bladeConfig.displayName - } - ) - - vmSubnetId: vmSubnetId - vmAdminUsername: vmAdminUsername - vmAdminPasswordOrKey: empty(vmAdminPasswordOrKey) ? existingVault.getSecret('PrivateLinkSSHKey-public') : vmAdminPasswordOrKey - workspaceName: workspaceName - authenticationType: empty(vmAdminPasswordOrKey) ? 'sshPublicKey' : 'password' - } -} diff --git a/bicep/modules/blade_network.bicep b/bicep/modules/blade_network.bicep index 50544412..265b984e 100644 --- a/bicep/modules/blade_network.bicep +++ b/bicep/modules/blade_network.bicep @@ -1,40 +1,6 @@ ///////////////// // Network Blade ///////////////// -// import * as type from 'types.bicep' - -type bladeSettings = { - @description('The name of the section name') - sectionName: string - @description('The display name of the section') - displayName: string -} - -type subnetSettings = { - @description('The name of the subnet') - name: string - @description('The address range to use for the subnet') - prefix: string -} - -type vnetSettings = { - @description('The name of the resource group that contains the Virtual Network') - group: string - @description('The name of the Virtual Network') - name: string - @description('The address range to use for the Virtual Network') - prefix: string - @description('The Managed Identity ') - identityId: string - @description('The cluster subnet') - aksSubnet: subnetSettings - @description('The pod subnet') - podSubnet: subnetSettings - @description('The machine subnet') - vmSubnet: subnetSettings - @description('The bastion subnet') - bastionSubnet: subnetSettings -} @description('The configuration for the blade section.') param bladeConfig bladeSettings @@ -55,7 +21,7 @@ param workspaceResourceId string param vnetConfiguration vnetSettings @description('Feature Flag to Enable Bastion') -param enableBastion bool +param enableBastion bool = false @description('Feature Flag to Enable a Pod Subnet') param enablePodSubnet bool @@ -420,3 +386,37 @@ output vnetId string = enableVnetInjection ? resourceId(networkConfiguration.gro output aksSubnetId string = enableVnetInjection ? '${resourceId(networkConfiguration.group, 'Microsoft.Network/virtualNetworks', networkConfiguration.name)}/subnets/${networkConfiguration.aksSubnet.name}' : '${network.outputs.resourceId}/subnets/${networkConfiguration.aksSubnet.name}' output vmSubnetId string = enableVnetInjection ? '${resourceId(networkConfiguration.group, 'Microsoft.Network/virtualNetworks', networkConfiguration.name)}/subnets/${networkConfiguration.vmSubnet.name}' : '${network.outputs.resourceId}/subnets/${networkConfiguration.vmSubnet.name}' output podSubnetId string = enableVnetInjection ? '${resourceId(networkConfiguration.group, 'Microsoft.Network/virtualNetworks', networkConfiguration.name)}/subnets/${networkConfiguration.podSubnet.name}' : '${network.outputs.resourceId}/subnets/${networkConfiguration.podSubnet.name}' + + +type bladeSettings = { + @description('The name of the section name') + sectionName: string + @description('The display name of the section') + displayName: string +} + +type subnetSettings = { + @description('The name of the subnet') + name: string + @description('The address range to use for the subnet') + prefix: string +} + +type vnetSettings = { + @description('The name of the resource group that contains the Virtual Network') + group: string + @description('The name of the Virtual Network') + name: string + @description('The address range to use for the Virtual Network') + prefix: string + @description('The Managed Identity ') + identityId: string + @description('The cluster subnet') + aksSubnet: subnetSettings + @description('The pod subnet') + podSubnet: subnetSettings + @description('The machine subnet') + vmSubnet: subnetSettings + @description('The bastion subnet') + bastionSubnet: subnetSettings +} diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index 21cdcf4c..5320b1f9 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -1,14 +1,6 @@ ///////////////// // Partition Blade ///////////////// -// import * as type from 'types.bicep' - -type bladeSettings = { - @description('The name of the section name') - sectionName: string - @description('The display name of the section') - displayName: string -} @description('The configuration for the blade section.') param bladeConfig bladeSettings @@ -47,13 +39,6 @@ param storageDNSZoneId string @description('Cosmos DNS Zone Id') param cosmosDNSZoneId string -@allowed([ - 'Burstable' - 'Standard' -]) -@description('The Partition Size') -param partitionSize string = 'Burstable' - @description('List of Data Partitions') param partitions array = [ { @@ -128,12 +113,7 @@ var partitionLayerConfig = { } database: { name: 'osdu-db' - Burstable : { - throughput: 2000 - } - Standard: { - throughput: 4000 - } + throughput: 4000 backup: 'Continuous' containers: [ { @@ -297,13 +277,6 @@ var partitionLayerConfig = { '/partitionKey' ] } - // { - // name: 'WorkflowTasksSharingInfoV2' - // kind: 'Hash' - // paths: [ - // '/partitionKey' - // ] - // } { name: 'WorkflowV2' kind: 'Hash' @@ -579,7 +552,7 @@ module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partiti array(partitionDatabase) ) : array(partitionDatabase) - maxThroughput: partitionLayerConfig.database[partitionSize].throughput + maxThroughput: partitionLayerConfig.database.throughput backupPolicyType: partitionLayerConfig.database.backup // Hookup Customer Managed Encryption Key @@ -704,3 +677,11 @@ module partitionSecrets './keyvault_secrets_partition.bicep' = [for (partition, // Output partitionStorage names output partitionStorageNames string[] = [for (partition, index) in partitions: partitionStorage[index].outputs.name] output partitionServiceBusNames string[] = [for (partition, index) in partitions: partitonNamespace[index].outputs.name] + + +type bladeSettings = { + @description('The name of the section name') + sectionName: string + @description('The display name of the section') + displayName: string +} diff --git a/bicep/modules/blade_service.bicep b/bicep/modules/blade_service.bicep index ffab1679..b3089fdc 100644 --- a/bicep/modules/blade_service.bicep +++ b/bicep/modules/blade_service.bicep @@ -1,25 +1,6 @@ ///////////////// // Service Blade ///////////////// -// import * as type from 'types.bicep' - -type bladeSettings = { - @description('The name of the section name') - sectionName: string - @description('The display name of the section') - displayName: string -} - -type appConfigItem = { - @description('The App Configuration Key') - name: string - @description('The App Configuration Value') - value: string - @description('The App Configuration Content Type') - contentType: string - @description('The App Configuration Label') - label: string -} @description('The configuration for the blade section.') param bladeConfig bladeSettings @@ -36,13 +17,8 @@ param enableTelemetry bool @description('The workspace resource Id for diagnostics') param workspaceResourceId string -param networkPlugin string - -@description('The size of the solution') -param clusterSize string - -@description('Override the default server type.') -param customVMSize string = '' +@description('A Custom VM Size for Internal Pool') +param vmSize string @description('The name of the Key Vault where the secret exists') param kvName string @@ -105,9 +81,6 @@ param enableAdminUI bool = true @description('Specify the OSDU version.') param osduVersion string = 'master' -@description('Optional: Specify the AD Users and/or Groups that can manage the cluster.') -param clusterAdminIds array - @minLength(9) @maxLength(18) @description('The address range to use for services') @@ -119,9 +92,10 @@ param serviceCidr string @description('The IP address to reserve for DNS') param dnsServiceIP string - +@description('The id of the subnet to deploy the AKS nodes') param aksSubnetId string +@description('The id of the subnet to deploy AKS pods') param podSubnetId string = '' @description('The managed identity name for deployment scripts') @@ -136,10 +110,12 @@ param partitionStorageNames string[] @description('The name of the partition service bus namespaces') param partitionServiceBusNames string[] - - -@description('Feature Flag to Enable Managed Observability.') -param enableMonitoring bool = false +@allowed([ + 'azureBlob' + 'gitRepository' +]) +@description('Flux source location for software definition.') +param sourceHost string = 'azureBlob' param appSettings appConfigItem[] @@ -156,9 +132,17 @@ var serviceLayerConfig = { sku: 'Basic' } cluster: { + tier: 'Standard' aksVersion: '1.30' meshVersion: 'asm-1-22' - networkPlugin: networkPlugin + + // D2pds v5 with 2 vCPUs and 8 GiB of memory. Available in 22 regions starting from $44.09 per month. + // D4pds v5 with 4 vCPUs and 16 GiB of memory. Available in 22 regions starting from $88.18 per month. + // D2s_v5 with 2 vCPUs and 8 GiB of memory. Available in 50 regions starting from $70.08 per month. + // D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month. + vmSize: 'Standard_D4pds_v5' + poolSize: 'Standard_D2pds_v5' + defaultSize: 'Standard_D4s_v5' // OSDU Java Services don't run on ARM? } gitops: { name: 'flux-system' @@ -168,6 +152,7 @@ var serviceLayerConfig = { components: './stamp/components' applications: './stamp/applications' experimental: './stamp/experimental' + enablePrivateSoftware: sourceHost == 'azureBlob' } } @@ -201,8 +186,6 @@ resource keySecret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { | _| `._____||_______| \______| |__| |_______/ |__| | _| `._____| |__| */ - - module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { name: '${bladeConfig.sectionName}-container-registry' params: { @@ -257,19 +240,13 @@ module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { |__|\__\ \______/ |______/ |_______|| _| `._____||__| \__| |_______| |__| |_______|_______/ */ -// This is used to help determine what the default subnet should be for the Pods if vnet injected is enabled -// var defaultPodSubnetId = enableVnetInjection ? '${resourceId(networkConfiguration.group, 'Microsoft.Network/virtualNetworks', networkConfiguration.name)}/subnets/${podSubnetName}' : '${network.outputs.resourceId}/subnets/${podSubnetName}' - -module cluster './aks_cluster.bicep' = { +module cluster './managed-cluster/main.bicep' = { name: '${bladeConfig.sectionName}-aks-cluster' params: { - // Basic Details - resourceName: bladeConfig.sectionName + name: '${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' location: location - aksVersion: serviceLayerConfig.cluster.aksVersion - aad_tenant_id: subscription().tenantId - clusterSize: clusterSize - networkPlugin: serviceLayerConfig.cluster.networkPlugin + skuTier: serviceLayerConfig.cluster.tier + kubernetesVersion: serviceLayerConfig.cluster.aksVersion // Assign Tags tags: union( @@ -279,110 +256,304 @@ module cluster './aks_cluster.bicep' = { } ) - // Configure Linking Items - aksSubnetId: aksSubnetId - aksPodSubnetId: podSubnetId == '' ? null : podSubnetId - identityId: identityId - workspaceId: workspaceResourceId + enableTelemetry: enableTelemetry + + // Hook up Diagnostics + diagnosticSettings: [ + { + workspaceResourceId: workspaceResourceId + } + ] + + roleAssignments: [ + { + roleDefinitionIdOrName: 'Azure Kubernetes Service RBAC Cluster Admin' + principalId: appIdentity.properties.principalId + principalType: 'ServicePrincipal' + } + // Role Assignment required for different extensions. + { + roleDefinitionIdOrName: 'Kubernetes Agentless Operator' + principalId: appIdentity.properties.principalId + principalType: 'ServicePrincipal' + } + ] + + aksServicePrincipalProfile: { + clientId: 'msi' + } + managedIdentities: { + systemAssigned: false + userAssignedResourcesIds: [ + identityId + ] + } + + // Network Settings + networkPlugin: 'azure' + networkPluginMode: empty(podSubnetId) ? 'overlay' : null + networkDataplane: 'cilium' + publicNetworkAccess: 'Enabled' + outboundType: empty(aksSubnetId) ? 'managedNATGateway' : 'loadBalancer' + enablePrivateCluster: false + + // Access Settings + disableLocalAccounts: true + enableRBAC: true + aadProfileManaged: true + nodeResourceGroupLockDown: true + + // Observability Settings + enableAzureDefender: true + enableContainerInsights: true + monitoringWorkspaceId: workspaceResourceId + enableAzureMonitorProfileMetrics: true + costAnalysisEnabled: true + + // Ingress Settings + webApplicationRoutingEnabled: false + openServiceMeshEnabled: false // Configure VNET Injection serviceCidr: serviceCidr dnsServiceIP: dnsServiceIP - // Configure Istio - serviceMeshProfile: enableMonitoring ? 'Istio' : null - istioRevision: enableMonitoring ? serviceLayerConfig.cluster.meshVersion : null - istioIngressGatewayMode: enableMonitoring ? clusterIngress : null - - // Configure Add Ons - enable_aad: empty(clusterAdminIds) == true ? false : true - admin_ids: clusterAdminIds - workloadIdentityEnabled: true - oidcIssuer: true - keyvaultEnabled: true - fluxGitOpsAddon: true + // Plugin Software + enableStorageProfileDiskCSIDriver: true + enableStorageProfileFileCSIDriver: true + enableStorageProfileSnapshotController: true + enableStorageProfileBlobCSIDriver: true + enableKeyvaultSecretsProvider: true + enableSecretRotation: true enableImageCleaner: true - fileCSIDriver: true - blobCSIDriver: true - azurepolicy: 'audit' - } -} + imageCleanerIntervalHours: 168 + enableOidcIssuerProfile: true + enableWorkloadIdentity: true + azurePolicyEnabled: true + omsAgentEnabled: true + + // Auto-Scaling + vpaAddon: true + kedaAddon: true + enableNodeAutoProvisioning: false + + maintenanceConfiguration: { + maintenanceWindow: { + schedule: { + daily: null + weekly: { + intervalWeeks: 1 + dayOfWeek: 'Sunday' + } + absoluteMonthly: null + relativeMonthly: null + } + durationHours: 4 + utcOffset: '+00:00' + startDate: '2024-10-01' + startTime: '00:00' + } + } + primaryAgentPoolProfile: [ + { + name: 'system' + mode: 'System' + vmSize: empty(vmSize) ? serviceLayerConfig.cluster.vmSize : vmSize + enableAutoScaling: true + minCount: 2 + maxCount: 6 + securityProfile: { + sshAccess: 'Disabled' + } + osType: 'Linux' + osSKU: 'AzureLinux' + availabilityZones: [ + '1' + '2' + '3' + ] + vnetSubnetID: !empty(aksSubnetId) ? aksSubnetId : null + podSubnetId: !empty(podSubnetId) ? podSubnetId : null + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + } + ] + // Additional Agent Pool Configurations + agentPools: [ + { + name: 'default' + mode: 'User' + vmSize: empty(vmSize) ? serviceLayerConfig.cluster.defaultSize : vmSize + enableAutoScaling: true + minCount: 4 + maxCount: 20 + sshAccess: 'Disabled' + osType: 'Linux' + osSku: 'AzureLinux' + availabilityZones: [ + '1' + '2' + '3' + ] + vnetSubnetID: !empty(aksSubnetId) ? aksSubnetId : null + podSubnetId: !empty(podSubnetId) ? podSubnetId : null + } + { + name: 'poolz1' + mode: 'User' + vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize + enableAutoScaling: true + minCount: 1 + maxCount: 3 + sshAccess: 'Disabled' + osType: 'Linux' + osSku: 'AzureLinux' + availabilityZones: [ + '1' + ] + vnetSubnetID: aksSubnetId + podSubnetId: podSubnetId + nodeTaints: ['app=cluster:NoSchedule'] + nodeLabels: { + app: 'cluster' + } + } + { + name: 'poolz2' + mode: 'User' + vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize + enableAutoScaling: true + minCount: 1 + maxCount: 3 + sshAccess: 'Disabled' + osType: 'Linux' + osSku: 'AzureLinux' + availabilityZones: [ + '2' + ] + vnetSubnetID: aksSubnetId + podSubnetId: podSubnetId + nodeTaints: ['app=cluster:NoSchedule'] + nodeLabels: { + app: 'cluster' + } + } + { + name: 'poolz3' + mode: 'User' + vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize + enableAutoScaling: true + minCount: 1 + maxCount: 3 + sshAccess: 'Disabled' + osType: 'Linux' + osSku: 'AzureLinux' + availabilityZones: [ + '3' + ] + vnetSubnetID: aksSubnetId + podSubnetId: podSubnetId + nodeTaints: ['app=cluster:NoSchedule'] + nodeLabels: { + app: 'cluster' + } + } + ] -///////////////// -// Elastic Configuration -///////////////// -var elasticPoolPresets = { - Burstable : { - vmSize: 'Standard_B4ms' } - Standard : { - vmSize: 'Standard_D4s_v5' +} + +// Policy Assignments custom module to apply the policies to the cluster. +module policy './managed-cluster/aks_policy.bicep' = { + name: '${bladeConfig.sectionName}-aks-policy' + params: { + clusterName: cluster.outputs.name } + dependsOn: [ + cluster + ] } -module pool1 './aks_agent_pool.bicep' = { - name: '${bladeConfig.sectionName}-pool1' +// AKS Extensions custom module to apply the app config provider extension to the cluster. +module appConfigExtension './managed-cluster/aks_appconfig_extension.bicep' = { + name: '${bladeConfig.sectionName}-aks-extension-appconfig' params: { - AksName: cluster.outputs.aksClusterName - PoolName: 'poolz1' - agentVMSize: empty(customVMSize) ? elasticPoolPresets[clusterSize].vmSize : customVMSize - agentCount: 1 - agentCountMax: 3 - availabilityZones: [ - '1' - ] - subnetId: aksSubnetId - podSubnetId: podSubnetId - nodeTaints: ['app=cluster:NoSchedule'] - nodeLabels: { - app: 'cluster' - } + clusterName: cluster.outputs.name } + dependsOn: [ + cluster + ] } -module pool2 './aks_agent_pool.bicep' = { - name: '${bladeConfig.sectionName}-pool2' +// AVM doesn't support output of the principalId from the extension module so we have to use a deployment script to get it. +module fluxExtension './flux-extension/main.bicep' = { + name: '${bladeConfig.sectionName}-flux-extension' params: { - AksName: cluster.outputs.aksClusterName - PoolName: 'poolz2' - agentVMSize: empty(customVMSize) ? elasticPoolPresets[clusterSize].vmSize : customVMSize - agentCount: 1 - agentCountMax: 3 - availabilityZones: [ - '2' - ] - subnetId: aksSubnetId - podSubnetId: podSubnetId - nodeTaints: ['app=cluster:NoSchedule'] - nodeLabels: { - app: 'cluster' + clusterName: cluster.outputs.name + location: location + extensionType: 'microsoft.flux' + name: 'flux' + releaseNamespace: 'flux-system' + releaseTrain: 'Stable' + + configurationSettings: { + 'multiTenancy.enforce': 'false' + 'helm-controller.enabled': 'true' + 'source-controller.enabled': 'true' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' } } } -module pool3 './aks_agent_pool.bicep' = { - name: '${bladeConfig.sectionName}-pool3' +module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = if (serviceLayerConfig.gitops.enablePrivateSoftware) { + name: '${bladeConfig.sectionName}-script-clientId' + params: { - AksName: cluster.outputs.aksClusterName - PoolName: 'poolz3' - agentVMSize: empty(customVMSize) ? elasticPoolPresets[clusterSize].vmSize : customVMSize - agentCount: 1 - agentCountMax: 3 - availabilityZones: [ - '3' - ] - subnetId: aksSubnetId - podSubnetId: podSubnetId - nodeTaints: ['app=cluster:NoSchedule'] - nodeLabels: { - app: 'cluster' + kind: 'AzureCLI' + name: 'aksExtensionClientId' + azCliVersion: '2.63.0' + location: location + managedIdentities: { + userAssignedResourcesIds: [ + appIdentity.id + ] } + + environmentVariables: [ + { + name: 'rgName' + value: '${resourceGroup().name}_aks_${cluster.outputs.name}_nodes' + } + { + name: 'principalId' + value: fluxExtension.outputs.principalId + } + ] + + timeout: 'PT30M' + retentionInterval: 'PT1H' + + scriptContent: ''' + az login --identity + + echo "Looking up client ID for $principalId in ResourceGroup $rgName" + clientId=$(az identity list --resource-group $rgName --query "[?principalId=='$principalId'] | [0].clientId" -otsv) + + echo "Found ClientId: $clientId" + echo "{\"clientId\":\"$clientId\"}" | jq -c '.' > $AZ_SCRIPTS_OUTPUT_PATH + ''' } + dependsOn: [ + fluxExtension + ] } - ///////////////// // Federated ///////////////// @@ -437,7 +608,7 @@ module federatedCredentials './federated_identity.bicep' = [for (cred, index) in audiences: [ 'api://AzureADTokenExchange' ] - issuer: cluster.outputs.aksOidcIssuerUrl + issuer: cluster.outputs.oidcIssuerUrl userAssignedIdentityName: appIdentity.name subject: cred.subject } @@ -469,28 +640,6 @@ module appRoleAssignments2 './app_assignments.bicep' = [for (name, index) in par ] }] -///////////////// -// Helm Charts -///////////////// -module helmAppConfigProvider './aks-run-command/main.bicep' = { - name: '${bladeConfig.sectionName}-helm-appconfig-provider' - params: { - aksName: cluster.outputs.aksClusterName - location: location - initialScriptDelay: '130s' - - newOrExistingManagedIdentity: 'existing' - managedIdentityName: managedIdentityName - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName:resourceGroup().name - - commands: [ - 'helm install azureappconfiguration.kubernetesprovider oci://mcr.microsoft.com/azure-app-configuration/helmchart/kubernetes-provider --namespace azappconfig-system --create-namespace' - ] - } -} - - /* ___ .______ .______ ______ ______ .__ __. _______ __ _______ @@ -681,7 +830,7 @@ values.yaml: | module appConfigMap './aks-config-map/main.bicep' = { name: '${bladeConfig.sectionName}-cluster-appconfig-configmap' params: { - aksName: cluster.outputs.aksClusterName + aksName: cluster.outputs.name location: location name: 'config-map-values' namespace: 'default' @@ -717,14 +866,16 @@ module appConfigMap './aks-config-map/main.bicep' = { \______| |__| |__| \______/ | _| |_______/ */ + + //--------------Flux Config--------------- module fluxConfiguration 'br/public:avm/res/kubernetes-configuration/flux-configuration:0.3.3' = if(enableSoftwareLoad) { name: '${bladeConfig.sectionName}-cluster-gitops' params: { name: serviceLayerConfig.gitops.name location: location - namespace: cluster.outputs.fluxReleaseNamespace - clusterName: cluster.outputs.aksClusterName + namespace: 'flux-system' + clusterName: cluster.outputs.name scope: 'cluster' sourceKind: 'GitRepository' gitRepository: { @@ -787,66 +938,10 @@ module fluxConfiguration 'br/public:avm/res/kubernetes-configuration/flux-config dependsOn: [ app_config appConfigMap - pool1 - pool2 - pool3 ] } -/* -.___ ___. ______ .__ __. __ .___________. ______ .______ -| \/ | / __ \ | \ | | | | | | / __ \ | _ \ -| \ / | | | | | | \| | | | `---| |----`| | | | | |_) | -| |\/| | | | | | | . ` | | | | | | | | | | / -| | | | | `--' | | |\ | | | | | | `--' | | |\ \----. -|__| |__| \______/ |__| \__| |__| |__| \______/ | _| `._____| -*/ - -var name = 'amw${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' - -module prometheus 'aks_prometheus.bicep' = if(enableMonitoring) { - name: '${bladeConfig.sectionName}-managed-prometheus' - params: { - // Basic Details - name: length(name) > 23 ? substring(name, 0, 23) : name - location: location - tags: union( - tags, - { - layer: bladeConfig.displayName - } - ) - - publicNetworkAccess: 'Enabled' - clusterName: cluster.outputs.aksClusterName - actionGroupId: '' - } -} - -module grafana 'aks_grafana.bicep' = if(enableMonitoring){ - name: '${bladeConfig.sectionName}-managed-grafana' - params: { - // Basic Details - name: length(name) > 23 ? substring(name, 0, 23) : name - location: location - tags: union( - tags, - { - layer: bladeConfig.displayName - } - ) - - skuName: 'Standard' - apiKey: 'Enabled' - autoGeneratedDomainNameLabelScope: 'TenantReuse' - deterministicOutboundIP: 'Disabled' - publicNetworkAccess: 'Enabled' - zoneRedundancy: 'Disabled' - prometheusName: prometheus.outputs.name - } -} - @description('The name of the container registry.') output registryName string = registry.outputs.name @@ -854,4 +949,24 @@ output registryName string = registry.outputs.name output appConfigName string = app_config.outputs.name @description('The name of the cluster.') -output clusterName string = cluster.outputs.aksClusterName +output clusterName string = cluster.outputs.name + + + +type bladeSettings = { + @description('The name of the section name') + sectionName: string + @description('The display name of the section') + displayName: string +} + +type appConfigItem = { + @description('The App Configuration Key') + name: string + @description('The App Configuration Value') + value: string + @description('The App Configuration Content Type') + contentType: string + @description('The App Configuration Label') + label: string +} diff --git a/bicep/modules/flux-configuration/README.md b/bicep/modules/flux-configuration/README.md new file mode 100644 index 00000000..57c4931d --- /dev/null +++ b/bicep/modules/flux-configuration/README.md @@ -0,0 +1,628 @@ +# Kubernetes Configuration Flux Configurations `[Microsoft.KubernetesConfiguration/fluxConfigurations]` + +This module deploys a Kubernetes Configuration Flux Configuration. + +## Navigation + +- [Resource Types](#Resource-Types) +- [Usage examples](#Usage-examples) +- [Parameters](#Parameters) +- [Outputs](#Outputs) +- [Notes](#Notes) +- [Data Collection](#Data-Collection) + +## Resource Types + +| Resource Type | API Version | +| :-- | :-- | +| `Microsoft.KubernetesConfiguration/fluxConfigurations` | [2023-05-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/fluxConfigurations) | + +## Usage examples + +The following section provides usage examples for the module, which were used to validate and deploy the module successfully. For a full reference, please review the module's test folder in its repository. + +>**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. + +>**Note**: To reference the module, please use the following syntax `br/public:avm/res/kubernetes-configuration/flux-configuration:`. + +- [Using only defaults](#example-1-using-only-defaults) +- [Using large parameter set](#example-2-using-large-parameter-set) +- [WAF-aligned](#example-3-waf-aligned) + +### Example 1: _Using only defaults_ + +This instance deploys the module with the minimum set of required parameters. + + +
+ +via Bicep module + +```bicep +module fluxConfiguration 'br/public:avm/res/kubernetes-configuration/flux-configuration:' = { + name: 'fluxConfigurationDeployment' + params: { + // Required parameters + clusterName: '' + kustomizations: { + unified: { + path: './cluster-manifests' + } + } + name: 'kcfcmin001' + namespace: 'flux-system' + scope: 'cluster' + sourceKind: 'GitRepository' + // Non-required parameters + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + location: '' + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "clusterName": { + "value": "" + }, + "kustomizations": { + "value": { + "unified": { + "path": "./cluster-manifests" + } + } + }, + "name": { + "value": "kcfcmin001" + }, + "namespace": { + "value": "flux-system" + }, + "scope": { + "value": "cluster" + }, + "sourceKind": { + "value": "GitRepository" + }, + // Non-required parameters + "gitRepository": { + "value": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + } + }, + "location": { + "value": "" + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/kubernetes-configuration/flux-configuration:' + +// Required parameters +param clusterName = '' +param kustomizations = { + unified: { + path: './cluster-manifests' + } +} +param name = 'kcfcmin001' +param namespace = 'flux-system' +param scope = 'cluster' +param sourceKind = 'GitRepository' +// Non-required parameters +param gitRepository = { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' +} +param location = '' +``` + +
+

+ +### Example 2: _Using large parameter set_ + +This instance deploys the module with most of its features enabled. + + +

+ +via Bicep module + +```bicep +module fluxConfiguration 'br/public:avm/res/kubernetes-configuration/flux-configuration:' = { + name: 'fluxConfigurationDeployment' + params: { + // Required parameters + clusterName: '' + kustomizations: { + unified: { + dependsOn: [] + force: false + path: './cluster-manifests' + postBuild: { + substitute: { + TEST_VAR1: 'foo' + TEST_VAR2: 'bar' + } + } + prune: true + syncIntervalInSeconds: 300 + timeoutInSeconds: 300 + } + } + name: 'kcfcmax001' + namespace: 'flux-system' + scope: 'cluster' + sourceKind: 'GitRepository' + // Non-required parameters + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + location: '' + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "clusterName": { + "value": "" + }, + "kustomizations": { + "value": { + "unified": { + "dependsOn": [], + "force": false, + "path": "./cluster-manifests", + "postBuild": { + "substitute": { + "TEST_VAR1": "foo", + "TEST_VAR2": "bar" + } + }, + "prune": true, + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 300 + } + } + }, + "name": { + "value": "kcfcmax001" + }, + "namespace": { + "value": "flux-system" + }, + "scope": { + "value": "cluster" + }, + "sourceKind": { + "value": "GitRepository" + }, + // Non-required parameters + "gitRepository": { + "value": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + } + }, + "location": { + "value": "" + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/kubernetes-configuration/flux-configuration:' + +// Required parameters +param clusterName = '' +param kustomizations = { + unified: { + dependsOn: [] + force: false + path: './cluster-manifests' + postBuild: { + substitute: { + TEST_VAR1: 'foo' + TEST_VAR2: 'bar' + } + } + prune: true + syncIntervalInSeconds: 300 + timeoutInSeconds: 300 + } +} +param name = 'kcfcmax001' +param namespace = 'flux-system' +param scope = 'cluster' +param sourceKind = 'GitRepository' +// Non-required parameters +param gitRepository = { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' +} +param location = '' +``` + +
+

+ +### Example 3: _WAF-aligned_ + +This instance deploys the module in alignment with the best-practices of the Azure Well-Architected Framework. + + +

+ +via Bicep module + +```bicep +module fluxConfiguration 'br/public:avm/res/kubernetes-configuration/flux-configuration:' = { + name: 'fluxConfigurationDeployment' + params: { + // Required parameters + clusterName: '' + kustomizations: { + unified: { + dependsOn: [] + force: false + path: './cluster-manifests' + prune: true + syncIntervalInSeconds: 300 + timeoutInSeconds: 300 + } + } + name: 'kcfcwaf001' + namespace: 'flux-system' + scope: 'cluster' + sourceKind: 'GitRepository' + // Non-required parameters + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + location: '' + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "clusterName": { + "value": "" + }, + "kustomizations": { + "value": { + "unified": { + "dependsOn": [], + "force": false, + "path": "./cluster-manifests", + "prune": true, + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 300 + } + } + }, + "name": { + "value": "kcfcwaf001" + }, + "namespace": { + "value": "flux-system" + }, + "scope": { + "value": "cluster" + }, + "sourceKind": { + "value": "GitRepository" + }, + // Non-required parameters + "gitRepository": { + "value": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + } + }, + "location": { + "value": "" + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/kubernetes-configuration/flux-configuration:' + +// Required parameters +param clusterName = '' +param kustomizations = { + unified: { + dependsOn: [] + force: false + path: './cluster-manifests' + prune: true + syncIntervalInSeconds: 300 + timeoutInSeconds: 300 + } +} +param name = 'kcfcwaf001' +param namespace = 'flux-system' +param scope = 'cluster' +param sourceKind = 'GitRepository' +// Non-required parameters +param gitRepository = { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' +} +param location = '' +``` + +
+

+ +## Parameters + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`clusterName`](#parameter-clustername) | string | The name of the AKS cluster that should be configured. | +| [`kustomizations`](#parameter-kustomizations) | object | Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster. | +| [`name`](#parameter-name) | string | The name of the Flux Configuration. | +| [`namespace`](#parameter-namespace) | string | The namespace to which this configuration is installed to. Maximum of 253 lower case alphanumeric characters, hyphen and period only. | +| [`scope`](#parameter-scope) | string | Scope at which the configuration will be installed. | +| [`sourceKind`](#parameter-sourcekind) | string | Source Kind to pull the configuration data from. | + +**Conditional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`bucket`](#parameter-bucket) | object | Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `Bucket`. | +| [`gitRepository`](#parameter-gitrepository) | object | Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `GitRepository`. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`configurationProtectedSettings`](#parameter-configurationprotectedsettings) | secureObject | Key-value pairs of protected configuration settings for the configuration. | +| [`enableTelemetry`](#parameter-enabletelemetry) | bool | Enable/Disable usage telemetry for module. | +| [`location`](#parameter-location) | string | Location for all resources. | +| [`suspend`](#parameter-suspend) | bool | Whether this configuration should suspend its reconciliation of its kustomizations and sources. | + +### Parameter: `clusterName` + +The name of the AKS cluster that should be configured. + +- Required: Yes +- Type: string + +### Parameter: `kustomizations` + +Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster. + +- Required: Yes +- Type: object + +### Parameter: `name` + +The name of the Flux Configuration. + +- Required: Yes +- Type: string + +### Parameter: `namespace` + +The namespace to which this configuration is installed to. Maximum of 253 lower case alphanumeric characters, hyphen and period only. + +- Required: Yes +- Type: string + +### Parameter: `scope` + +Scope at which the configuration will be installed. + +- Required: Yes +- Type: string +- Allowed: + ```Bicep + [ + 'cluster' + 'namespace' + ] + ``` + +### Parameter: `sourceKind` + +Source Kind to pull the configuration data from. + +- Required: Yes +- Type: string +- Allowed: + ```Bicep + [ + 'Bucket' + 'GitRepository' + ] + ``` + +### Parameter: `bucket` + +Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `Bucket`. + +- Required: No +- Type: object + +### Parameter: `gitRepository` + +Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `GitRepository`. + +- Required: No +- Type: object + +### Parameter: `configurationProtectedSettings` + +Key-value pairs of protected configuration settings for the configuration. + +- Required: No +- Type: secureObject + +### Parameter: `enableTelemetry` + +Enable/Disable usage telemetry for module. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `location` + +Location for all resources. + +- Required: No +- Type: string +- Default: `[resourceGroup().location]` + +### Parameter: `suspend` + +Whether this configuration should suspend its reconciliation of its kustomizations and sources. + +- Required: No +- Type: bool +- Default: `False` + +## Outputs + +| Output | Type | Description | +| :-- | :-- | :-- | +| `name` | string | The name of the flux configuration. | +| `resourceGroupName` | string | The name of the resource group the flux configuration was deployed into. | +| `resourceId` | string | The resource ID of the flux configuration. | + +## Notes + +To use this module, it is required to register the AKS-ExtensionManager feature in your subscription. To do so, you can use the following command: + +```powershell +az feature register --namespace Microsoft.ContainerService --name AKS-ExtensionManager +``` + +Further it is required to register the following Azure service providers. (It's OK to re-register an existing provider.) + +```powershell +az provider register --namespace Microsoft.Kubernetes +az provider register --namespace Microsoft.ContainerService +az provider register --namespace Microsoft.KubernetesConfiguration +``` + +For more details see [Prerequisites](https://learn.microsoft.com/en-us/azure/azure-arc/kubernetes/tutorial-use-gitops-flux2) + +## Data Collection + +The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the [repository](https://aka.ms/avm/telemetry). There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft’s privacy statement. Our privacy statement is located at . You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. diff --git a/bicep/modules/flux-configuration/main.bicep b/bicep/modules/flux-configuration/main.bicep new file mode 100644 index 00000000..82c636ab --- /dev/null +++ b/bicep/modules/flux-configuration/main.bicep @@ -0,0 +1,100 @@ +metadata name = 'Kubernetes Configuration Flux Configurations' +metadata description = 'This module deploys a Kubernetes Configuration Flux Configuration.' +metadata owner = 'Azure/module-maintainers' + +@description('Required. The name of the Flux Configuration.') +param name string + +@description('Optional. Enable/Disable usage telemetry for module.') +param enableTelemetry bool = true + +@description('Required. The name of the AKS cluster that should be configured.') +param clusterName string + +@description('Optional. Location for all resources.') +param location string = resourceGroup().location + +@description('Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `Bucket`.') +param bucket object? + +@description('Conditional. Parameters to reconcile to the AzureBlob source kind type. Required if `sourceKind` is `AzureBlob`.') +param azureBlob object? + +@description('Optional. Key-value pairs of protected configuration settings for the configuration.') +@secure() +param configurationProtectedSettings object? + +@description('Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `GitRepository`.') +param gitRepository object? + +@description('Required. Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster.') +param kustomizations object + +@description('Required. The namespace to which this configuration is installed to. Maximum of 253 lower case alphanumeric characters, hyphen and period only.') +param namespace string + +@allowed([ + 'cluster' + 'namespace' +]) +@description('Required. Scope at which the configuration will be installed.') +param scope string + +@allowed([ + 'Bucket' + 'GitRepository' + 'AzureBlob' +]) +@description('Required. Source Kind to pull the configuration data from.') +param sourceKind string + +@description('Optional. Whether this configuration should suspend its reconciliation of its kustomizations and sources.') +param suspend bool = false + +#disable-next-line no-deployments-resources +resource avmTelemetry 'Microsoft.Resources/deployments@2024-03-01' = if (enableTelemetry) { + name: '46d3xbcp.res.kubernetesconfiguration-fluxconfig.${replace('-..--..-', '.', '-')}.${substring(uniqueString(deployment().name, location), 0, 4)}' + properties: { + mode: 'Incremental' + template: { + '$schema': 'https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#' + contentVersion: '1.0.0.0' + resources: [] + outputs: { + telemetry: { + type: 'String' + value: 'For more information, see https://aka.ms/avm/TelemetryInfo' + } + } + } + } +} + +resource managedCluster 'Microsoft.ContainerService/managedClusters@2022-07-01' existing = { + name: clusterName +} + +resource fluxConfiguration 'Microsoft.KubernetesConfiguration/fluxConfigurations@2024-04-01-preview' = { + name: name + scope: managedCluster + properties: { + bucket: bucket + azureBlob: azureBlob + configurationProtectedSettings: configurationProtectedSettings + gitRepository: gitRepository + kustomizations: kustomizations + namespace: namespace + scope: scope + sourceKind: sourceKind + suspend: suspend + } +} + +@description('The name of the flux configuration.') +output name string = fluxConfiguration.name + +@description('The resource ID of the flux configuration.') +output resourceId string = fluxConfiguration.id + +@description('The name of the resource group the flux configuration was deployed into.') +output resourceGroupName string = resourceGroup().name diff --git a/bicep/modules/flux-configuration/main.json b/bicep/modules/flux-configuration/main.json new file mode 100644 index 00000000..ff775093 --- /dev/null +++ b/bicep/modules/flux-configuration/main.json @@ -0,0 +1,173 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.28.1.47646", + "templateHash": "542320603270173721" + }, + "name": "Kubernetes Configuration Flux Configurations", + "description": "This module deploys a Kubernetes Configuration Flux Configuration.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. The name of the Flux Configuration." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "clusterName": { + "type": "string", + "metadata": { + "description": "Required. The name of the AKS cluster that should be configured." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Location for all resources." + } + }, + "bucket": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `Bucket`." + } + }, + "configurationProtectedSettings": { + "type": "secureObject", + "nullable": true, + "metadata": { + "description": "Optional. Key-value pairs of protected configuration settings for the configuration." + } + }, + "gitRepository": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `GitRepository`." + } + }, + "kustomizations": { + "type": "object", + "metadata": { + "description": "Required. Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster." + } + }, + "namespace": { + "type": "string", + "metadata": { + "description": "Required. The namespace to which this configuration is installed to. Maximum of 253 lower case alphanumeric characters, hyphen and period only." + } + }, + "scope": { + "type": "string", + "allowedValues": [ + "cluster", + "namespace" + ], + "metadata": { + "description": "Required. Scope at which the configuration will be installed." + } + }, + "sourceKind": { + "type": "string", + "allowedValues": [ + "Bucket", + "GitRepository" + ], + "metadata": { + "description": "Required. Source Kind to pull the configuration data from." + } + }, + "suspend": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether this configuration should suspend its reconciliation of its kustomizations and sources." + } + } + }, + "resources": { + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2024-03-01", + "name": "[format('46d3xbcp.res.kubernetesconfiguration-fluxconfig.{0}.{1}', replace('-..--..-', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2022-07-01", + "name": "[parameters('clusterName')]" + }, + "fluxConfiguration": { + "type": "Microsoft.KubernetesConfiguration/fluxConfigurations", + "apiVersion": "2023-05-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('clusterName'))]", + "name": "[parameters('name')]", + "properties": { + "bucket": "[parameters('bucket')]", + "configurationProtectedSettings": "[parameters('configurationProtectedSettings')]", + "gitRepository": "[parameters('gitRepository')]", + "kustomizations": "[parameters('kustomizations')]", + "namespace": "[parameters('namespace')]", + "scope": "[parameters('scope')]", + "sourceKind": "[parameters('sourceKind')]", + "suspend": "[parameters('suspend')]" + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the flux configuration." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the flux configuration." + }, + "value": "[extensionResourceId(resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), 'Microsoft.KubernetesConfiguration/fluxConfigurations', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The name of the resource group the flux configuration was deployed into." + }, + "value": "[resourceGroup().name]" + } + } +} \ No newline at end of file diff --git a/bicep/modules/flux-configuration/tests/e2e/defaults/dependencies.bicep b/bicep/modules/flux-configuration/tests/e2e/defaults/dependencies.bicep new file mode 100644 index 00000000..0bf942bb --- /dev/null +++ b/bicep/modules/flux-configuration/tests/e2e/defaults/dependencies.bicep @@ -0,0 +1,49 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the AKS cluster to create.') +param clusterName string + +@description('Required. The name of the AKS cluster extension to create.') +param clusterExtensionName string + +@description('Required. The name of the AKS cluster nodes resource group to create.') +param clusterNodeResourceGroupName string + +resource cluster 'Microsoft.ContainerService/managedClusters@2022-07-01' = { + name: clusterName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + dnsPrefix: clusterName + nodeResourceGroup: clusterNodeResourceGroupName + agentPoolProfiles: [ + { + name: 'agentpool' + count: 1 + vmSize: 'Standard_DS2_v2' + osType: 'Linux' + mode: 'System' + } + ] + } +} + +resource extension 'Microsoft.KubernetesConfiguration/extensions@2022-03-01' = { + scope: cluster + name: clusterExtensionName + properties: { + extensionType: 'microsoft.flux' + releaseTrain: 'Stable' + scope: { + cluster: { + releaseNamespace: 'flux-system' + } + } + } +} + +@description('The name of the created AKS cluster.') +output clusterName string = cluster.name diff --git a/bicep/modules/flux-configuration/tests/e2e/defaults/main.test.bicep b/bicep/modules/flux-configuration/tests/e2e/defaults/main.test.bicep new file mode 100644 index 00000000..430f6d59 --- /dev/null +++ b/bicep/modules/flux-configuration/tests/e2e/defaults/main.test.bicep @@ -0,0 +1,80 @@ +targetScope = 'subscription' + +metadata name = 'Using only defaults' +metadata description = 'This instance deploys the module with the minimum set of required parameters.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-kubernetesconfiguration.fluxconfigurations-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'kcfcmin' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + clusterName: 'dep-${namePrefix}-aks-${serviceShort}' + clusterExtensionName: '${namePrefix}${serviceShort}001' + clusterNodeResourceGroupName: 'dep-${namePrefix}-aks-${serviceShort}-rg' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + location: resourceLocation + name: '${namePrefix}${serviceShort}001' + clusterName: nestedDependencies.outputs.clusterName + namespace: 'flux-system' + scope: 'cluster' + sourceKind: 'GitRepository' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + path: './cluster-manifests' + } + } + } + dependsOn: [ + nestedDependencies + ] + } +] diff --git a/bicep/modules/flux-configuration/tests/e2e/max/dependencies.bicep b/bicep/modules/flux-configuration/tests/e2e/max/dependencies.bicep new file mode 100644 index 00000000..0bf942bb --- /dev/null +++ b/bicep/modules/flux-configuration/tests/e2e/max/dependencies.bicep @@ -0,0 +1,49 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the AKS cluster to create.') +param clusterName string + +@description('Required. The name of the AKS cluster extension to create.') +param clusterExtensionName string + +@description('Required. The name of the AKS cluster nodes resource group to create.') +param clusterNodeResourceGroupName string + +resource cluster 'Microsoft.ContainerService/managedClusters@2022-07-01' = { + name: clusterName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + dnsPrefix: clusterName + nodeResourceGroup: clusterNodeResourceGroupName + agentPoolProfiles: [ + { + name: 'agentpool' + count: 1 + vmSize: 'Standard_DS2_v2' + osType: 'Linux' + mode: 'System' + } + ] + } +} + +resource extension 'Microsoft.KubernetesConfiguration/extensions@2022-03-01' = { + scope: cluster + name: clusterExtensionName + properties: { + extensionType: 'microsoft.flux' + releaseTrain: 'Stable' + scope: { + cluster: { + releaseNamespace: 'flux-system' + } + } + } +} + +@description('The name of the created AKS cluster.') +output clusterName string = cluster.name diff --git a/bicep/modules/flux-configuration/tests/e2e/max/main.test.bicep b/bicep/modules/flux-configuration/tests/e2e/max/main.test.bicep new file mode 100644 index 00000000..0545b6e5 --- /dev/null +++ b/bicep/modules/flux-configuration/tests/e2e/max/main.test.bicep @@ -0,0 +1,91 @@ +targetScope = 'subscription' + +metadata name = 'Using large parameter set' +metadata description = 'This instance deploys the module with most of its features enabled.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-kubernetesconfiguration.fluxconfigurations-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'kcfcmax' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + clusterName: 'dep-${namePrefix}-aks-${serviceShort}' + clusterExtensionName: '${namePrefix}${serviceShort}001' + clusterNodeResourceGroupName: 'dep-${namePrefix}-aks-${serviceShort}-rg' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + clusterName: nestedDependencies.outputs.clusterName + namespace: 'flux-system' + scope: 'cluster' + sourceKind: 'GitRepository' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + dependsOn: [] + force: false + path: './cluster-manifests' + prune: true + syncIntervalInSeconds: 300 + timeoutInSeconds: 300 + postBuild: { + substitute: { + TEST_VAR1: 'foo' + TEST_VAR2: 'bar' + } + } + } + } + } + dependsOn: [ + nestedDependencies + ] + } +] diff --git a/bicep/modules/flux-configuration/tests/e2e/waf-aligned/dependencies.bicep b/bicep/modules/flux-configuration/tests/e2e/waf-aligned/dependencies.bicep new file mode 100644 index 00000000..0bf942bb --- /dev/null +++ b/bicep/modules/flux-configuration/tests/e2e/waf-aligned/dependencies.bicep @@ -0,0 +1,49 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the AKS cluster to create.') +param clusterName string + +@description('Required. The name of the AKS cluster extension to create.') +param clusterExtensionName string + +@description('Required. The name of the AKS cluster nodes resource group to create.') +param clusterNodeResourceGroupName string + +resource cluster 'Microsoft.ContainerService/managedClusters@2022-07-01' = { + name: clusterName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + dnsPrefix: clusterName + nodeResourceGroup: clusterNodeResourceGroupName + agentPoolProfiles: [ + { + name: 'agentpool' + count: 1 + vmSize: 'Standard_DS2_v2' + osType: 'Linux' + mode: 'System' + } + ] + } +} + +resource extension 'Microsoft.KubernetesConfiguration/extensions@2022-03-01' = { + scope: cluster + name: clusterExtensionName + properties: { + extensionType: 'microsoft.flux' + releaseTrain: 'Stable' + scope: { + cluster: { + releaseNamespace: 'flux-system' + } + } + } +} + +@description('The name of the created AKS cluster.') +output clusterName string = cluster.name diff --git a/bicep/modules/flux-configuration/tests/e2e/waf-aligned/main.test.bicep b/bicep/modules/flux-configuration/tests/e2e/waf-aligned/main.test.bicep new file mode 100644 index 00000000..10516046 --- /dev/null +++ b/bicep/modules/flux-configuration/tests/e2e/waf-aligned/main.test.bicep @@ -0,0 +1,85 @@ +targetScope = 'subscription' + +metadata name = 'WAF-aligned' +metadata description = 'This instance deploys the module in alignment with the best-practices of the Azure Well-Architected Framework.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-kubernetesconfiguration.fluxconfigurations-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'kcfcwaf' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + clusterName: 'dep-${namePrefix}-aks-${serviceShort}' + clusterExtensionName: '${namePrefix}${serviceShort}001' + clusterNodeResourceGroupName: 'dep-${namePrefix}-aks-${serviceShort}-rg' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + clusterName: nestedDependencies.outputs.clusterName + namespace: 'flux-system' + location: resourceLocation + scope: 'cluster' + sourceKind: 'GitRepository' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + dependsOn: [] + force: false + path: './cluster-manifests' + prune: true + syncIntervalInSeconds: 300 + timeoutInSeconds: 300 + } + } + } + dependsOn: [ + nestedDependencies + ] + } +] diff --git a/bicep/modules/flux-configuration/tests/unit/custom.tests.ps1 b/bicep/modules/flux-configuration/tests/unit/custom.tests.ps1 new file mode 100644 index 00000000..36b53af9 --- /dev/null +++ b/bicep/modules/flux-configuration/tests/unit/custom.tests.ps1 @@ -0,0 +1,7 @@ +########################### +## Additional unit tests ## +########################### +## +## You can add any custom static validation tests you want here, or add them spread accross multiple test files in the unit folder. +## +########################### diff --git a/bicep/modules/flux-configuration/version.json b/bicep/modules/flux-configuration/version.json new file mode 100644 index 00000000..c177b1bb --- /dev/null +++ b/bicep/modules/flux-configuration/version.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", + "version": "0.3", + "pathFilters": [ + "./main.json" + ] +} \ No newline at end of file diff --git a/bicep/modules/flux-extension/README.md b/bicep/modules/flux-extension/README.md new file mode 100644 index 00000000..2467e5fb --- /dev/null +++ b/bicep/modules/flux-extension/README.md @@ -0,0 +1,618 @@ +# Kubernetes Configuration Extensions `[Microsoft.KubernetesConfiguration/extensions]` + +This module deploys a Kubernetes Configuration Extension. + +## Navigation + +- [Resource Types](#Resource-Types) +- [Usage examples](#Usage-examples) +- [Parameters](#Parameters) +- [Outputs](#Outputs) +- [Cross-referenced modules](#Cross-referenced-modules) +- [Notes](#Notes) +- [Data Collection](#Data-Collection) + +## Resource Types + +| Resource Type | API Version | +| :-- | :-- | +| `Microsoft.KubernetesConfiguration/extensions` | [2022-03-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/extensions) | +| `Microsoft.KubernetesConfiguration/fluxConfigurations` | [2023-05-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/fluxConfigurations) | + +## Usage examples + +The following section provides usage examples for the module, which were used to validate and deploy the module successfully. For a full reference, please review the module's test folder in its repository. + +>**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. + +>**Note**: To reference the module, please use the following syntax `br/public:avm/res/kubernetes-configuration/extension:`. + +- [Using only defaults](#example-1-using-only-defaults) +- [Using large parameter set](#example-2-using-large-parameter-set) +- [WAF-aligned](#example-3-waf-aligned) + +### Example 1: _Using only defaults_ + +This instance deploys the module with the minimum set of required parameters. + + +

+ +via Bicep module + +```bicep +module extension 'br/public:avm/res/kubernetes-configuration/extension:' = { + name: 'extensionDeployment' + params: { + // Required parameters + clusterName: '' + extensionType: 'microsoft.flux' + name: 'kcemin001' + // Non-required parameters + location: '' + releaseNamespace: 'flux-system' + releaseTrain: 'Stable' + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "clusterName": { + "value": "" + }, + "extensionType": { + "value": "microsoft.flux" + }, + "name": { + "value": "kcemin001" + }, + // Non-required parameters + "location": { + "value": "" + }, + "releaseNamespace": { + "value": "flux-system" + }, + "releaseTrain": { + "value": "Stable" + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/kubernetes-configuration/extension:' + +// Required parameters +param clusterName = '' +param extensionType = 'microsoft.flux' +param name = 'kcemin001' +// Non-required parameters +param location = '' +param releaseNamespace = 'flux-system' +param releaseTrain = 'Stable' +``` + +
+

+ +### Example 2: _Using large parameter set_ + +This instance deploys the module with most of its features enabled. + + +

+ +via Bicep module + +```bicep +module extension 'br/public:avm/res/kubernetes-configuration/extension:' = { + name: 'extensionDeployment' + params: { + // Required parameters + clusterName: '' + extensionType: 'microsoft.flux' + name: 'kcemax001' + // Non-required parameters + configurationSettings: { + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'false' + 'source-controller.enabled': 'true' + } + fluxConfigurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + path: './cluster-manifests' + } + } + namespace: 'flux-system' + scope: 'cluster' + suspend: false + } + ] + location: '' + releaseNamespace: 'flux-system' + releaseTrain: 'Stable' + version: '0.5.2' + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "clusterName": { + "value": "" + }, + "extensionType": { + "value": "microsoft.flux" + }, + "name": { + "value": "kcemax001" + }, + // Non-required parameters + "configurationSettings": { + "value": { + "image-automation-controller.enabled": "false", + "image-reflector-controller.enabled": "false", + "kustomize-controller.enabled": "true", + "notification-controller.enabled": "false", + "source-controller.enabled": "true" + } + }, + "fluxConfigurations": { + "value": [ + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + }, + "kustomizations": { + "unified": { + "path": "./cluster-manifests" + } + }, + "namespace": "flux-system", + "scope": "cluster", + "suspend": false + } + ] + }, + "location": { + "value": "" + }, + "releaseNamespace": { + "value": "flux-system" + }, + "releaseTrain": { + "value": "Stable" + }, + "version": { + "value": "0.5.2" + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/kubernetes-configuration/extension:' + +// Required parameters +param clusterName = '' +param extensionType = 'microsoft.flux' +param name = 'kcemax001' +// Non-required parameters +param configurationSettings = { + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'false' + 'source-controller.enabled': 'true' +} +param fluxConfigurations = [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + path: './cluster-manifests' + } + } + namespace: 'flux-system' + scope: 'cluster' + suspend: false + } +] +param location = '' +param releaseNamespace = 'flux-system' +param releaseTrain = 'Stable' +param version = '0.5.2' +``` + +
+

+ +### Example 3: _WAF-aligned_ + +This instance deploys the module in alignment with the best-practices of the Azure Well-Architected Framework. + + +

+ +via Bicep module + +```bicep +module extension 'br/public:avm/res/kubernetes-configuration/extension:' = { + name: 'extensionDeployment' + params: { + // Required parameters + clusterName: '' + extensionType: 'microsoft.flux' + name: 'kcewaf001' + // Non-required parameters + configurationSettings: { + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'false' + 'source-controller.enabled': 'true' + } + fluxConfigurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + path: './cluster-manifests' + } + } + namespace: 'flux-system' + scope: 'cluster' + suspend: false + } + ] + location: '' + releaseNamespace: 'flux-system' + releaseTrain: 'Stable' + version: '0.5.2' + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "clusterName": { + "value": "" + }, + "extensionType": { + "value": "microsoft.flux" + }, + "name": { + "value": "kcewaf001" + }, + // Non-required parameters + "configurationSettings": { + "value": { + "image-automation-controller.enabled": "false", + "image-reflector-controller.enabled": "false", + "kustomize-controller.enabled": "true", + "notification-controller.enabled": "false", + "source-controller.enabled": "true" + } + }, + "fluxConfigurations": { + "value": [ + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + }, + "kustomizations": { + "unified": { + "path": "./cluster-manifests" + } + }, + "namespace": "flux-system", + "scope": "cluster", + "suspend": false + } + ] + }, + "location": { + "value": "" + }, + "releaseNamespace": { + "value": "flux-system" + }, + "releaseTrain": { + "value": "Stable" + }, + "version": { + "value": "0.5.2" + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/kubernetes-configuration/extension:' + +// Required parameters +param clusterName = '' +param extensionType = 'microsoft.flux' +param name = 'kcewaf001' +// Non-required parameters +param configurationSettings = { + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'false' + 'source-controller.enabled': 'true' +} +param fluxConfigurations = [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + path: './cluster-manifests' + } + } + namespace: 'flux-system' + scope: 'cluster' + suspend: false + } +] +param location = '' +param releaseNamespace = 'flux-system' +param releaseTrain = 'Stable' +param version = '0.5.2' +``` + +
+

+ +## Parameters + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`clusterName`](#parameter-clustername) | string | The name of the AKS cluster that should be configured. | +| [`extensionType`](#parameter-extensiontype) | string | Type of the extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the extension publisher. | +| [`name`](#parameter-name) | string | The name of the Flux Configuration. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`configurationProtectedSettings`](#parameter-configurationprotectedsettings) | secureObject | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | +| [`configurationSettings`](#parameter-configurationsettings) | object | Configuration settings, as name-value pairs for configuring this extension. | +| [`enableTelemetry`](#parameter-enabletelemetry) | bool | Enable/Disable usage telemetry for module. | +| [`fluxConfigurations`](#parameter-fluxconfigurations) | array | A list of flux configuraitons. | +| [`location`](#parameter-location) | string | Location for all resources. | +| [`releaseNamespace`](#parameter-releasenamespace) | string | Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created. | +| [`releaseTrain`](#parameter-releasetrain) | string | ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true". | +| [`targetNamespace`](#parameter-targetnamespace) | string | Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created. | +| [`version`](#parameter-version) | string | Version of the extension for this extension, if it is "pinned" to a specific version. | + +### Parameter: `clusterName` + +The name of the AKS cluster that should be configured. + +- Required: Yes +- Type: string + +### Parameter: `extensionType` + +Type of the extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the extension publisher. + +- Required: Yes +- Type: string + +### Parameter: `name` + +The name of the Flux Configuration. + +- Required: Yes +- Type: string + +### Parameter: `configurationProtectedSettings` + +Configuration settings that are sensitive, as name-value pairs for configuring this extension. + +- Required: No +- Type: secureObject + +### Parameter: `configurationSettings` + +Configuration settings, as name-value pairs for configuring this extension. + +- Required: No +- Type: object + +### Parameter: `enableTelemetry` + +Enable/Disable usage telemetry for module. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `fluxConfigurations` + +A list of flux configuraitons. + +- Required: No +- Type: array + +### Parameter: `location` + +Location for all resources. + +- Required: No +- Type: string +- Default: `[resourceGroup().location]` + +### Parameter: `releaseNamespace` + +Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created. + +- Required: No +- Type: string + +### Parameter: `releaseTrain` + +ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true". + +- Required: No +- Type: string + +### Parameter: `targetNamespace` + +Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created. + +- Required: No +- Type: string + +### Parameter: `version` + +Version of the extension for this extension, if it is "pinned" to a specific version. + +- Required: No +- Type: string + +## Outputs + +| Output | Type | Description | +| :-- | :-- | :-- | +| `name` | string | The name of the extension. | +| `resourceGroupName` | string | The name of the resource group the extension was deployed into. | +| `resourceId` | string | The resource ID of the extension. | + +## Cross-referenced modules + +This section gives you an overview of all local-referenced module files (i.e., other modules that are referenced in this module) and all remote-referenced files (i.e., Bicep modules that are referenced from a Bicep Registry or Template Specs). + +| Reference | Type | +| :-- | :-- | +| `br/public:avm/res/kubernetes-configuration/flux-configuration:0.3.1` | Remote reference | + +## Notes + +To use this module, it is required to register the AKS-ExtensionManager feature in your subscription. To do so, you can use the following command: + +```powershell +az feature register --namespace Microsoft.ContainerService --name AKS-ExtensionManager +``` + +Further it is required to register the following Azure service providers. (It's OK to re-register an existing provider.) + +```powershell +az provider register --namespace Microsoft.Kubernetes +az provider register --namespace Microsoft.ContainerService +az provider register --namespace Microsoft.KubernetesConfiguration +``` + +For more details see [Prerequisites](https://learn.microsoft.com/en-us/azure/azure-arc/kubernetes/tutorial-use-gitops-flux2) + +## Data Collection + +The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the [repository](https://aka.ms/avm/telemetry). There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft’s privacy statement. Our privacy statement is located at . You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. diff --git a/bicep/modules/flux-extension/main.bicep b/bicep/modules/flux-extension/main.bicep new file mode 100644 index 00000000..86f1181b --- /dev/null +++ b/bicep/modules/flux-extension/main.bicep @@ -0,0 +1,121 @@ +metadata name = 'Kubernetes Configuration Extensions' +metadata description = 'This module deploys a Kubernetes Configuration Extension.' +metadata owner = 'Azure/module-maintainers' + +@description('Required. The name of the Flux Configuration.') +param name string + +@description('Optional. Enable/Disable usage telemetry for module.') +param enableTelemetry bool = true + +@description('Required. The name of the AKS cluster that should be configured.') +param clusterName string + +@description('Optional. Location for all resources.') +param location string = resourceGroup().location + +@description('Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension.') +@secure() +param configurationProtectedSettings object? + +@description('Optional. Configuration settings, as name-value pairs for configuring this extension.') +param configurationSettings object? + +@description('Required. Type of the extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the extension publisher.') +param extensionType string + +@description('Optional. ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true".') +param releaseTrain string? + +@description('Optional. Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created.') +param releaseNamespace string? + +@description('Optional. Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created.') +param targetNamespace string? + +@description('Optional. Version of the extension for this extension, if it is "pinned" to a specific version.') +param version string? + +@description('Optional. A list of flux configuraitons.') +param fluxConfigurations array? + +#disable-next-line no-deployments-resources +resource avmTelemetry 'Microsoft.Resources/deployments@2024-03-01' = if (enableTelemetry) { + name: '46d3xbcp.res.kubernetesconfiguration-extension.${replace('-..--..-', '.', '-')}.${substring(uniqueString(deployment().name, location), 0, 4)}' + properties: { + mode: 'Incremental' + template: { + '$schema': 'https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#' + contentVersion: '1.0.0.0' + resources: [] + outputs: { + telemetry: { + type: 'String' + value: 'For more information, see https://aka.ms/avm/TelemetryInfo' + } + } + } + } +} + +resource managedCluster 'Microsoft.ContainerService/managedClusters@2022-07-01' existing = { + name: clusterName +} + +resource extension 'Microsoft.KubernetesConfiguration/extensions@2022-03-01' = { + name: name + scope: managedCluster + properties: { + autoUpgradeMinorVersion: !empty(version) ? false : true + configurationProtectedSettings: configurationProtectedSettings + configurationSettings: configurationSettings + extensionType: extensionType + releaseTrain: releaseTrain + scope: { + cluster: !empty(releaseNamespace ?? '') + ? { + releaseNamespace: releaseNamespace + } + : null + namespace: !empty(targetNamespace ?? '') + ? { + targetNamespace: targetNamespace + } + : null + } + version: version + } +} + +module fluxConfiguration 'br/public:avm/res/kubernetes-configuration/flux-configuration:0.3.1' = [ + for (fluxConfiguration, index) in (fluxConfigurations ?? []): { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxConfiguration${index}' + params: { + enableTelemetry: fluxConfiguration.?enableTelemetry ?? enableTelemetry + clusterName: managedCluster.name + scope: fluxConfiguration.scope + namespace: fluxConfiguration.namespace + sourceKind: contains(fluxConfiguration, 'gitRepository') ? 'GitRepository' : 'Bucket' + name: fluxConfiguration.?name ?? toLower('${managedCluster.name}-fluxconfiguration${index}') + bucket: fluxConfiguration.?bucket + configurationProtectedSettings: fluxConfiguration.?configurationProtectedSettings + gitRepository: fluxConfiguration.?gitRepository + kustomizations: fluxConfiguration.kustomizations + suspend: fluxConfiguration.?suspend + } + dependsOn: [ + extension + ] + } +] + +@description('The name of the extension.') +output name string = extension.name + +@description('The resource ID of the extension.') +output resourceId string = extension.id + +output principalId string = extension.properties.aksAssignedIdentity.principalId + +@description('The name of the resource group the extension was deployed into.') +output resourceGroupName string = resourceGroup().name diff --git a/bicep/modules/flux-extension/main.json b/bicep/modules/flux-extension/main.json new file mode 100644 index 00000000..1609f862 --- /dev/null +++ b/bicep/modules/flux-extension/main.json @@ -0,0 +1,395 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.28.1.47646", + "templateHash": "1500434239147541571" + }, + "name": "Kubernetes Configuration Extensions", + "description": "This module deploys a Kubernetes Configuration Extension.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. The name of the Flux Configuration." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "clusterName": { + "type": "string", + "metadata": { + "description": "Required. The name of the AKS cluster that should be configured." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Location for all resources." + } + }, + "configurationProtectedSettings": { + "type": "secureObject", + "nullable": true, + "metadata": { + "description": "Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension." + } + }, + "configurationSettings": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Configuration settings, as name-value pairs for configuring this extension." + } + }, + "extensionType": { + "type": "string", + "metadata": { + "description": "Required. Type of the extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the extension publisher." + } + }, + "releaseTrain": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is \"true\"." + } + }, + "releaseNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created." + } + }, + "targetNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created." + } + }, + "version": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Version of the extension for this extension, if it is \"pinned\" to a specific version." + } + }, + "fluxConfigurations": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. A list of flux configuraitons." + } + } + }, + "resources": { + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2024-03-01", + "name": "[format('46d3xbcp.res.kubernetesconfiguration-extension.{0}.{1}', replace('-..--..-', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2022-07-01", + "name": "[parameters('clusterName')]" + }, + "extension": { + "type": "Microsoft.KubernetesConfiguration/extensions", + "apiVersion": "2022-03-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('clusterName'))]", + "name": "[parameters('name')]", + "properties": { + "autoUpgradeMinorVersion": "[if(not(empty(parameters('version'))), false(), true())]", + "configurationProtectedSettings": "[parameters('configurationProtectedSettings')]", + "configurationSettings": "[parameters('configurationSettings')]", + "extensionType": "[parameters('extensionType')]", + "releaseTrain": "[parameters('releaseTrain')]", + "scope": { + "cluster": "[if(not(empty(coalesce(parameters('releaseNamespace'), ''))), createObject('releaseNamespace', parameters('releaseNamespace')), null())]", + "namespace": "[if(not(empty(coalesce(parameters('targetNamespace'), ''))), createObject('targetNamespace', parameters('targetNamespace')), null())]" + }, + "version": "[parameters('version')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "fluxConfiguration": { + "copy": { + "name": "fluxConfiguration", + "count": "[length(coalesce(parameters('fluxConfigurations'), createArray()))]" + }, + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-ManagedCluster-FluxConfiguration{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "enableTelemetry": { + "value": "[coalesce(tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'enableTelemetry'), parameters('enableTelemetry'))]" + }, + "clusterName": { + "value": "[parameters('clusterName')]" + }, + "scope": { + "value": "[coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()].scope]" + }, + "namespace": { + "value": "[coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()].namespace]" + }, + "sourceKind": "[if(contains(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'gitRepository'), createObject('value', 'GitRepository'), createObject('value', 'Bucket'))]", + "name": { + "value": "[coalesce(tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'name'), toLower(format('{0}-fluxconfiguration{1}', parameters('clusterName'), copyIndex())))]" + }, + "bucket": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'bucket')]" + }, + "configurationProtectedSettings": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'configurationProtectedSettings')]" + }, + "gitRepository": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'gitRepository')]" + }, + "kustomizations": { + "value": "[coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()].kustomizations]" + }, + "suspend": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'suspend')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.23.1.45101", + "templateHash": "885928168160399718" + }, + "name": "Kubernetes Configuration Flux Configurations", + "description": "This module deploys a Kubernetes Configuration Flux Configuration.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. The name of the Flux Configuration." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "clusterName": { + "type": "string", + "metadata": { + "description": "Required. The name of the AKS cluster that should be configured." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Location for all resources." + } + }, + "bucket": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `Bucket`." + } + }, + "configurationProtectedSettings": { + "type": "secureObject", + "nullable": true, + "metadata": { + "description": "Optional. Key-value pairs of protected configuration settings for the configuration." + } + }, + "gitRepository": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `GitRepository`." + } + }, + "kustomizations": { + "type": "object", + "metadata": { + "description": "Required. Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster." + } + }, + "namespace": { + "type": "string", + "metadata": { + "description": "Required. The namespace to which this configuration is installed to. Maximum of 253 lower case alphanumeric characters, hyphen and period only." + } + }, + "scope": { + "type": "string", + "allowedValues": [ + "cluster", + "namespace" + ], + "metadata": { + "description": "Required. Scope at which the configuration will be installed." + } + }, + "sourceKind": { + "type": "string", + "allowedValues": [ + "Bucket", + "GitRepository" + ], + "metadata": { + "description": "Required. Source Kind to pull the configuration data from." + } + }, + "suspend": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether this configuration should suspend its reconciliation of its kustomizations and sources." + } + } + }, + "resources": { + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2023-07-01", + "name": "[format('46d3xbcp.res.kubernetesconfiguration-fluxconfig.{0}.{1}', replace('0.3.1', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2022-07-01", + "name": "[parameters('clusterName')]" + }, + "fluxConfiguration": { + "type": "Microsoft.KubernetesConfiguration/fluxConfigurations", + "apiVersion": "2023-05-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('clusterName'))]", + "name": "[parameters('name')]", + "properties": { + "bucket": "[parameters('bucket')]", + "configurationProtectedSettings": "[parameters('configurationProtectedSettings')]", + "gitRepository": "[parameters('gitRepository')]", + "kustomizations": "[parameters('kustomizations')]", + "namespace": "[parameters('namespace')]", + "scope": "[parameters('scope')]", + "sourceKind": "[parameters('sourceKind')]", + "suspend": "[parameters('suspend')]" + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the flux configuration." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the flux configuration." + }, + "value": "[extensionResourceId(resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), 'Microsoft.KubernetesConfiguration/fluxConfigurations', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The name of the resource group the flux configuration was deployed into." + }, + "value": "[resourceGroup().name]" + } + } + } + }, + "dependsOn": [ + "extension", + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the extension." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the extension." + }, + "value": "[extensionResourceId(resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), 'Microsoft.KubernetesConfiguration/extensions', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The name of the resource group the extension was deployed into." + }, + "value": "[resourceGroup().name]" + } + } +} \ No newline at end of file diff --git a/bicep/modules/flux-extension/tests/e2e/defaults/dependencies.bicep b/bicep/modules/flux-extension/tests/e2e/defaults/dependencies.bicep new file mode 100644 index 00000000..01697635 --- /dev/null +++ b/bicep/modules/flux-extension/tests/e2e/defaults/dependencies.bicep @@ -0,0 +1,32 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the AKS cluster to create.') +param clusterName string + +@description('Required. The name of the AKS cluster nodes resource group to create.') +param clusterNodeResourceGroupName string + +resource cluster 'Microsoft.ContainerService/managedClusters@2022-07-01' = { + name: clusterName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + dnsPrefix: clusterName + nodeResourceGroup: clusterNodeResourceGroupName + agentPoolProfiles: [ + { + name: 'agentpool' + count: 1 + vmSize: 'Standard_DS2_v2' + osType: 'Linux' + mode: 'System' + } + ] + } +} + +@description('The name of the created AKS cluster.') +output clusterName string = cluster.name diff --git a/bicep/modules/flux-extension/tests/e2e/defaults/main.test.bicep b/bicep/modules/flux-extension/tests/e2e/defaults/main.test.bicep new file mode 100644 index 00000000..b46932fa --- /dev/null +++ b/bicep/modules/flux-extension/tests/e2e/defaults/main.test.bicep @@ -0,0 +1,64 @@ +targetScope = 'subscription' + +metadata name = 'Using only defaults' +metadata description = 'This instance deploys the module with the minimum set of required parameters.' +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-kubernetesconfiguration.extensions-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'kcemin' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + clusterName: 'dep-${namePrefix}-aks-${serviceShort}' + clusterNodeResourceGroupName: 'nodes-${resourceGroupName}' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + clusterName: nestedDependencies.outputs.clusterName + extensionType: 'microsoft.flux' + releaseNamespace: 'flux-system' + releaseTrain: 'Stable' + } + dependsOn: [ + nestedDependencies + ] + } +] diff --git a/bicep/modules/flux-extension/tests/e2e/max/dependencies.bicep b/bicep/modules/flux-extension/tests/e2e/max/dependencies.bicep new file mode 100644 index 00000000..01697635 --- /dev/null +++ b/bicep/modules/flux-extension/tests/e2e/max/dependencies.bicep @@ -0,0 +1,32 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the AKS cluster to create.') +param clusterName string + +@description('Required. The name of the AKS cluster nodes resource group to create.') +param clusterNodeResourceGroupName string + +resource cluster 'Microsoft.ContainerService/managedClusters@2022-07-01' = { + name: clusterName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + dnsPrefix: clusterName + nodeResourceGroup: clusterNodeResourceGroupName + agentPoolProfiles: [ + { + name: 'agentpool' + count: 1 + vmSize: 'Standard_DS2_v2' + osType: 'Linux' + mode: 'System' + } + ] + } +} + +@description('The name of the created AKS cluster.') +output clusterName string = cluster.name diff --git a/bicep/modules/flux-extension/tests/e2e/max/main.test.bicep b/bicep/modules/flux-extension/tests/e2e/max/main.test.bicep new file mode 100644 index 00000000..835be36f --- /dev/null +++ b/bicep/modules/flux-extension/tests/e2e/max/main.test.bicep @@ -0,0 +1,94 @@ +targetScope = 'subscription' + +metadata name = 'Using large parameter set' +metadata description = 'This instance deploys the module with most of its features enabled.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-kubernetesconfiguration.extensions-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'kcemax' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + clusterName: 'dep-${namePrefix}-aks-${serviceShort}' + clusterNodeResourceGroupName: 'nodes-${resourceGroupName}' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + clusterName: nestedDependencies.outputs.clusterName + extensionType: 'microsoft.flux' + configurationSettings: { + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'false' + 'source-controller.enabled': 'true' + } + releaseNamespace: 'flux-system' + releaseTrain: 'Stable' + version: '0.5.2' + fluxConfigurations: [ + { + namespace: 'flux-system' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + path: './cluster-manifests' + } + } + suspend: false + } + ] + } + dependsOn: [ + nestedDependencies + ] + } +] diff --git a/bicep/modules/flux-extension/tests/e2e/waf-aligned/dependencies.bicep b/bicep/modules/flux-extension/tests/e2e/waf-aligned/dependencies.bicep new file mode 100644 index 00000000..01697635 --- /dev/null +++ b/bicep/modules/flux-extension/tests/e2e/waf-aligned/dependencies.bicep @@ -0,0 +1,32 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the AKS cluster to create.') +param clusterName string + +@description('Required. The name of the AKS cluster nodes resource group to create.') +param clusterNodeResourceGroupName string + +resource cluster 'Microsoft.ContainerService/managedClusters@2022-07-01' = { + name: clusterName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + dnsPrefix: clusterName + nodeResourceGroup: clusterNodeResourceGroupName + agentPoolProfiles: [ + { + name: 'agentpool' + count: 1 + vmSize: 'Standard_DS2_v2' + osType: 'Linux' + mode: 'System' + } + ] + } +} + +@description('The name of the created AKS cluster.') +output clusterName string = cluster.name diff --git a/bicep/modules/flux-extension/tests/e2e/waf-aligned/main.test.bicep b/bicep/modules/flux-extension/tests/e2e/waf-aligned/main.test.bicep new file mode 100644 index 00000000..b9fb8d10 --- /dev/null +++ b/bicep/modules/flux-extension/tests/e2e/waf-aligned/main.test.bicep @@ -0,0 +1,94 @@ +targetScope = 'subscription' + +metadata name = 'WAF-aligned' +metadata description = 'This instance deploys the module in alignment with the best-practices of the Azure Well-Architected Framework.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-kubernetesconfiguration.extensions-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'kcewaf' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + clusterName: 'dep-${namePrefix}-aks-${serviceShort}' + clusterNodeResourceGroupName: 'nodes-${resourceGroupName}' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + clusterName: nestedDependencies.outputs.clusterName + extensionType: 'microsoft.flux' + configurationSettings: { + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'false' + 'source-controller.enabled': 'true' + } + releaseNamespace: 'flux-system' + releaseTrain: 'Stable' + version: '0.5.2' + fluxConfigurations: [ + { + namespace: 'flux-system' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + kustomizations: { + unified: { + path: './cluster-manifests' + } + } + suspend: false + } + ] + } + dependsOn: [ + nestedDependencies + ] + } +] diff --git a/bicep/modules/flux-extension/tests/unit/custom.tests.ps1 b/bicep/modules/flux-extension/tests/unit/custom.tests.ps1 new file mode 100644 index 00000000..36b53af9 --- /dev/null +++ b/bicep/modules/flux-extension/tests/unit/custom.tests.ps1 @@ -0,0 +1,7 @@ +########################### +## Additional unit tests ## +########################### +## +## You can add any custom static validation tests you want here, or add them spread accross multiple test files in the unit folder. +## +########################### diff --git a/bicep/modules/flux-extension/version.json b/bicep/modules/flux-extension/version.json new file mode 100644 index 00000000..c177b1bb --- /dev/null +++ b/bicep/modules/flux-extension/version.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", + "version": "0.3", + "pathFilters": [ + "./main.json" + ] +} \ No newline at end of file diff --git a/bicep/modules/managed-cluster/README.md b/bicep/modules/managed-cluster/README.md new file mode 100644 index 00000000..3b68c834 --- /dev/null +++ b/bicep/modules/managed-cluster/README.md @@ -0,0 +1,4050 @@ +# Azure Kubernetes Service (AKS) Managed Clusters `[Microsoft.ContainerService/managedClusters]` + +This module deploys an Azure Kubernetes Service (AKS) Managed Cluster. + +## Navigation + +- [Resource Types](#Resource-Types) +- [Usage examples](#Usage-examples) +- [Parameters](#Parameters) +- [Outputs](#Outputs) +- [Cross-referenced modules](#Cross-referenced-modules) +- [Data Collection](#Data-Collection) + +## Resource Types + +| Resource Type | API Version | +| :-- | :-- | +| `Microsoft.Authorization/locks` | [2020-05-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2020-05-01/locks) | +| `Microsoft.Authorization/roleAssignments` | [2022-04-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2022-04-01/roleAssignments) | +| `Microsoft.ContainerService/managedClusters` | [2024-03-02-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2024-03-02-preview/managedClusters) | +| `Microsoft.ContainerService/managedClusters/agentPools` | [2023-07-02-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2023-07-02-preview/managedClusters/agentPools) | +| `Microsoft.ContainerService/managedClusters/maintenanceConfigurations` | [2023-10-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2023-10-01/managedClusters/maintenanceConfigurations) | +| `Microsoft.Insights/diagnosticSettings` | [2021-05-01-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Insights/2021-05-01-preview/diagnosticSettings) | +| `Microsoft.KubernetesConfiguration/extensions` | [2022-03-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/extensions) | +| `Microsoft.KubernetesConfiguration/fluxConfigurations` | [2022-03-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/fluxConfigurations) | + +## Usage examples + +The following section provides usage examples for the module, which were used to validate and deploy the module successfully. For a full reference, please review the module's test folder in its repository. + +>**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. + +>**Note**: To reference the module, please use the following syntax `br/public:avm/res/container-service/managed-cluster:`. + +- [Using only defaults and use AKS Automatic mode](#example-1-using-only-defaults-and-use-aks-automatic-mode) +- [Using Azure CNI Network Plugin.](#example-2-using-azure-cni-network-plugin) +- [Using only defaults](#example-3-using-only-defaults) +- [Using Kubenet Network Plugin.](#example-4-using-kubenet-network-plugin) +- [Using Private Cluster.](#example-5-using-private-cluster) +- [WAF-aligned](#example-6-waf-aligned) + +### Example 1: _Using only defaults and use AKS Automatic mode_ + +This instance deploys the module with the set of automatic parameters. + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: 'managedClusterDeployment' + params: { + // Required parameters + name: 'csauto001' + primaryAgentPoolProfile: [ + { + count: 3 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + location: '' + maintenanceConfiguration: { + maintenanceWindow: { + durationHours: 4 + schedule: { + absoluteMonthly: '' + daily: '' + relativeMonthly: '' + weekly: { + dayOfWeek: 'Sunday' + intervalWeeks: 1 + } + } + startDate: '2024-07-03' + startTime: '00:00' + utcOffset: '+00:00' + } + } + managedIdentities: { + systemAssigned: true + } + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csauto001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "count": 3, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "location": { + "value": "" + }, + "maintenanceConfiguration": { + "value": { + "maintenanceWindow": { + "durationHours": 4, + "schedule": { + "absoluteMonthly": "", + "daily": "", + "relativeMonthly": "", + "weekly": { + "dayOfWeek": "Sunday", + "intervalWeeks": 1 + } + }, + "startDate": "2024-07-03", + "startTime": "00:00", + "utcOffset": "+00:00" + } + } + }, + "managedIdentities": { + "value": { + "systemAssigned": true + } + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/container-service/managed-cluster:' + +// Required parameters +param name = 'csauto001' +param primaryAgentPoolProfile = [ + { + count: 3 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } +] +// Non-required parameters +param location = '' +param maintenanceConfiguration = { + maintenanceWindow: { + durationHours: 4 + schedule: { + absoluteMonthly: '' + daily: '' + relativeMonthly: '' + weekly: { + dayOfWeek: 'Sunday' + intervalWeeks: 1 + } + } + startDate: '2024-07-03' + startTime: '00:00' + utcOffset: '+00:00' + } +} +param managedIdentities = { + systemAssigned: true +} +``` + +
+

+ +### Example 2: _Using Azure CNI Network Plugin._ + +This instance deploys the module with Azure CNI network plugin . + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: 'managedClusterDeployment' + params: { + // Required parameters + name: 'csmaz001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + proximityPlacementGroupResourceId: '' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + customerManagedKey: { + keyName: '' + keyVaultNetworkAccess: 'Public' + keyVaultResourceId: '' + } + diagnosticSettings: [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } + ] + diskEncryptionSetResourceId: '' + enableAzureDefender: true + enableAzureMonitorProfileMetrics: true + enableKeyvaultSecretsProvider: true + enableOidcIssuerProfile: true + enablePodSecurityPolicy: false + enableStorageProfileBlobCSIDriver: true + enableStorageProfileDiskCSIDriver: true + enableStorageProfileFileCSIDriver: true + enableStorageProfileSnapshotController: true + enableWorkloadIdentity: true + fluxExtension: { + configurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + namespace: 'flux-system' + scope: 'cluster' + } + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + apps: { + dependsOn: [ + 'infra' + ] + path: './apps/staging' + prune: true + retryIntervalInSeconds: 120 + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + } + infra: { + dependsOn: [] + path: './infrastructure' + prune: true + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + validation: 'none' + } + } + namespace: 'flux-system-helm' + scope: 'cluster' + } + ] + configurationSettings: { + 'helm-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'source-controller.enabled': 'true' + } + } + identityProfile: { + kubeletidentity: { + resourceId: '' + } + } + location: '' + lock: { + kind: 'CanNotDelete' + name: 'myCustomLockName' + } + managedIdentities: { + userAssignedResourcesIds: [ + '' + ] + } + monitoringWorkspaceId: '' + networkDataplane: 'azure' + networkPlugin: 'azure' + networkPluginMode: 'overlay' + omsAgentEnabled: true + openServiceMeshEnabled: true + roleAssignments: [ + { + name: 'ac915208-669e-4665-9792-7e2dc861f569' + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'Owner' + } + { + name: '' + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: '' + } + ] + tags: { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' + } + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmaz001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 1, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "mode": "System", + "name": "systempool", + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 0, + "osType": "Linux", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "osDiskSizeGB": 128, + "osType": "Linux", + "proximityPlacementGroupResourceId": "", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + }, + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + "autoUpgradeProfileUpgradeChannel": { + "value": "stable" + }, + "customerManagedKey": { + "value": { + "keyName": "", + "keyVaultNetworkAccess": "Public", + "keyVaultResourceId": "" + } + }, + "diagnosticSettings": { + "value": [ + { + "eventHubAuthorizationRuleResourceId": "", + "eventHubName": "", + "metricCategories": [ + { + "category": "AllMetrics" + } + ], + "name": "customSetting", + "storageAccountResourceId": "", + "workspaceResourceId": "" + } + ] + }, + "diskEncryptionSetResourceId": { + "value": "" + }, + "enableAzureDefender": { + "value": true + }, + "enableAzureMonitorProfileMetrics": { + "value": true + }, + "enableKeyvaultSecretsProvider": { + "value": true + }, + "enableOidcIssuerProfile": { + "value": true + }, + "enablePodSecurityPolicy": { + "value": false + }, + "enableStorageProfileBlobCSIDriver": { + "value": true + }, + "enableStorageProfileDiskCSIDriver": { + "value": true + }, + "enableStorageProfileFileCSIDriver": { + "value": true + }, + "enableStorageProfileSnapshotController": { + "value": true + }, + "enableWorkloadIdentity": { + "value": true + }, + "fluxExtension": { + "value": { + "configurations": [ + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + }, + "namespace": "flux-system", + "scope": "cluster" + }, + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/Azure/gitops-flux2-kustomize-helm-mt" + }, + "kustomizations": { + "apps": { + "dependsOn": [ + "infra" + ], + "path": "./apps/staging", + "prune": true, + "retryIntervalInSeconds": 120, + "syncIntervalInSeconds": 600, + "timeoutInSeconds": 600 + }, + "infra": { + "dependsOn": [], + "path": "./infrastructure", + "prune": true, + "syncIntervalInSeconds": 600, + "timeoutInSeconds": 600, + "validation": "none" + } + }, + "namespace": "flux-system-helm", + "scope": "cluster" + } + ], + "configurationSettings": { + "helm-controller.enabled": "true", + "image-automation-controller.enabled": "false", + "image-reflector-controller.enabled": "false", + "kustomize-controller.enabled": "true", + "notification-controller.enabled": "true", + "source-controller.enabled": "true" + } + } + }, + "identityProfile": { + "value": { + "kubeletidentity": { + "resourceId": "" + } + } + }, + "location": { + "value": "" + }, + "lock": { + "value": { + "kind": "CanNotDelete", + "name": "myCustomLockName" + } + }, + "managedIdentities": { + "value": { + "userAssignedResourcesIds": [ + "" + ] + } + }, + "monitoringWorkspaceId": { + "value": "" + }, + "networkDataplane": { + "value": "azure" + }, + "networkPlugin": { + "value": "azure" + }, + "networkPluginMode": { + "value": "overlay" + }, + "omsAgentEnabled": { + "value": true + }, + "openServiceMeshEnabled": { + "value": true + }, + "roleAssignments": { + "value": [ + { + "name": "ac915208-669e-4665-9792-7e2dc861f569", + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "Owner" + }, + { + "name": "", + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "b24988ac-6180-42a0-ab88-20f7382dd24c" + }, + { + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "" + } + ] + }, + "tags": { + "value": { + "Environment": "Non-Prod", + "hidden-title": "This is visible in the resource name", + "Role": "DeploymentValidation" + } + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/container-service/managed-cluster:' + +// Required parameters +param name = 'csmaz001' +param primaryAgentPoolProfile = [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } +] +// Non-required parameters +param agentPools = [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + proximityPlacementGroupResourceId: '' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } +] +param autoUpgradeProfileUpgradeChannel = 'stable' +param customerManagedKey = { + keyName: '' + keyVaultNetworkAccess: 'Public' + keyVaultResourceId: '' +} +param diagnosticSettings = [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } +] +param diskEncryptionSetResourceId = '' +param enableAzureDefender = true +param enableAzureMonitorProfileMetrics = true +param enableKeyvaultSecretsProvider = true +param enableOidcIssuerProfile = true +param enablePodSecurityPolicy = false +param enableStorageProfileBlobCSIDriver = true +param enableStorageProfileDiskCSIDriver = true +param enableStorageProfileFileCSIDriver = true +param enableStorageProfileSnapshotController = true +param enableWorkloadIdentity = true +param fluxExtension = { + configurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + namespace: 'flux-system' + scope: 'cluster' + } + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + apps: { + dependsOn: [ + 'infra' + ] + path: './apps/staging' + prune: true + retryIntervalInSeconds: 120 + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + } + infra: { + dependsOn: [] + path: './infrastructure' + prune: true + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + validation: 'none' + } + } + namespace: 'flux-system-helm' + scope: 'cluster' + } + ] + configurationSettings: { + 'helm-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'source-controller.enabled': 'true' + } +} +param identityProfile = { + kubeletidentity: { + resourceId: '' + } +} +param location = '' +param lock = { + kind: 'CanNotDelete' + name: 'myCustomLockName' +} +param managedIdentities = { + userAssignedResourcesIds: [ + '' + ] +} +param monitoringWorkspaceId = '' +param networkDataplane = 'azure' +param networkPlugin = 'azure' +param networkPluginMode = 'overlay' +param omsAgentEnabled = true +param openServiceMeshEnabled = true +param roleAssignments = [ + { + name: 'ac915208-669e-4665-9792-7e2dc861f569' + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'Owner' + } + { + name: '' + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: '' + } +] +param tags = { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' +} +``` + +
+

+ +### Example 3: _Using only defaults_ + +This instance deploys the module with the minimum set of required parameters. + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: 'managedClusterDeployment' + params: { + // Required parameters + name: 'csmin001' + primaryAgentPoolProfile: [ + { + count: 3 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + location: '' + managedIdentities: { + systemAssigned: true + } + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmin001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "count": 3, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "systemAssigned": true + } + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/container-service/managed-cluster:' + +// Required parameters +param name = 'csmin001' +param primaryAgentPoolProfile = [ + { + count: 3 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } +] +// Non-required parameters +param location = '' +param managedIdentities = { + systemAssigned: true +} +``` + +
+

+ +### Example 4: _Using Kubenet Network Plugin._ + +This instance deploys the module with Kubenet network plugin . + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: 'managedClusterDeployment' + params: { + // Required parameters + name: 'csmkube001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + diagnosticSettings: [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } + ] + location: '' + managedIdentities: { + userAssignedResourcesIds: [ + '' + ] + } + networkPlugin: 'kubenet' + roleAssignments: [ + { + name: '6acf186b-abbd-491b-8bd7-39fa199da81e' + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'Owner' + } + { + name: '' + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: '' + } + ] + tags: { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' + } + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmkube001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 1, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "mode": "System", + "name": "systempool", + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 0, + "osType": "Linux", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + }, + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + "diagnosticSettings": { + "value": [ + { + "eventHubAuthorizationRuleResourceId": "", + "eventHubName": "", + "metricCategories": [ + { + "category": "AllMetrics" + } + ], + "name": "customSetting", + "storageAccountResourceId": "", + "workspaceResourceId": "" + } + ] + }, + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "userAssignedResourcesIds": [ + "" + ] + } + }, + "networkPlugin": { + "value": "kubenet" + }, + "roleAssignments": { + "value": [ + { + "name": "6acf186b-abbd-491b-8bd7-39fa199da81e", + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "Owner" + }, + { + "name": "", + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "b24988ac-6180-42a0-ab88-20f7382dd24c" + }, + { + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "" + } + ] + }, + "tags": { + "value": { + "Environment": "Non-Prod", + "hidden-title": "This is visible in the resource name", + "Role": "DeploymentValidation" + } + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/container-service/managed-cluster:' + +// Required parameters +param name = 'csmkube001' +param primaryAgentPoolProfile = [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } +] +// Non-required parameters +param agentPools = [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } +] +param diagnosticSettings = [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } +] +param location = '' +param managedIdentities = { + userAssignedResourcesIds: [ + '' + ] +} +param networkPlugin = 'kubenet' +param roleAssignments = [ + { + name: '6acf186b-abbd-491b-8bd7-39fa199da81e' + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'Owner' + } + { + name: '' + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: '' + } +] +param tags = { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' +} +``` + +
+

+ +### Example 5: _Using Private Cluster._ + +This instance deploys the module with a private cluster instance. + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: 'managedClusterDeployment' + params: { + // Required parameters + name: 'csmpriv001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + dnsServiceIP: '10.10.200.10' + enablePrivateCluster: true + location: '' + managedIdentities: { + userAssignedResourcesIds: [ + '' + ] + } + networkPlugin: 'azure' + privateDNSZone: '' + serviceCidr: '10.10.200.0/24' + skuTier: 'Standard' + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmpriv001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 1, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "mode": "System", + "name": "systempool", + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 0, + "osType": "Linux", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + }, + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + "dnsServiceIP": { + "value": "10.10.200.10" + }, + "enablePrivateCluster": { + "value": true + }, + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "userAssignedResourcesIds": [ + "" + ] + } + }, + "networkPlugin": { + "value": "azure" + }, + "privateDNSZone": { + "value": "" + }, + "serviceCidr": { + "value": "10.10.200.0/24" + }, + "skuTier": { + "value": "Standard" + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/container-service/managed-cluster:' + +// Required parameters +param name = 'csmpriv001' +param primaryAgentPoolProfile = [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } +] +// Non-required parameters +param agentPools = [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } +] +param dnsServiceIP = '10.10.200.10' +param enablePrivateCluster = true +param location = '' +param managedIdentities = { + userAssignedResourcesIds: [ + '' + ] +} +param networkPlugin = 'azure' +param privateDNSZone = '' +param serviceCidr = '10.10.200.0/24' +param skuTier = 'Standard' +``` + +
+

+ +### Example 6: _WAF-aligned_ + +This instance deploys the module in alignment with the best-practices of the Well-Architected Framework. + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: 'managedClusterDeployment' + params: { + // Required parameters + name: 'cswaf001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 60 + osDiskType: 'Ephemeral' + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 60 + osDiskType: 'Ephemeral' + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + diagnosticSettings: [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + logCategoriesAndGroups: [ + { + category: 'kube-apiserver' + } + { + category: 'kube-controller-manager' + } + { + category: 'kube-scheduler' + } + { + category: 'cluster-autoscaler' + } + ] + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } + ] + disableLocalAccounts: true + dnsServiceIP: '10.10.200.10' + enableAzureDefender: true + enablePrivateCluster: true + location: '' + managedIdentities: { + userAssignedResourcesIds: [ + '' + ] + } + monitoringWorkspaceId: '' + networkPlugin: 'azure' + networkPolicy: 'azure' + omsAgentEnabled: true + privateDNSZone: '' + serviceCidr: '10.10.200.0/24' + skuTier: 'Standard' + tags: { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' + } + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "cswaf001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 3, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 50, + "minCount": 3, + "mode": "System", + "name": "systempool", + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 0, + "osType": "Linux", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 3, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 50, + "minCount": 3, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "osDiskSizeGB": 60, + "osDiskType": "Ephemeral", + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + }, + { + "availabilityZones": [ + "3" + ], + "count": 3, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 50, + "minCount": 3, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "osDiskSizeGB": 60, + "osDiskType": "Ephemeral", + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + "autoUpgradeProfileUpgradeChannel": { + "value": "stable" + }, + "diagnosticSettings": { + "value": [ + { + "eventHubAuthorizationRuleResourceId": "", + "eventHubName": "", + "logCategoriesAndGroups": [ + { + "category": "kube-apiserver" + }, + { + "category": "kube-controller-manager" + }, + { + "category": "kube-scheduler" + }, + { + "category": "cluster-autoscaler" + } + ], + "metricCategories": [ + { + "category": "AllMetrics" + } + ], + "name": "customSetting", + "storageAccountResourceId": "", + "workspaceResourceId": "" + } + ] + }, + "disableLocalAccounts": { + "value": true + }, + "dnsServiceIP": { + "value": "10.10.200.10" + }, + "enableAzureDefender": { + "value": true + }, + "enablePrivateCluster": { + "value": true + }, + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "userAssignedResourcesIds": [ + "" + ] + } + }, + "monitoringWorkspaceId": { + "value": "" + }, + "networkPlugin": { + "value": "azure" + }, + "networkPolicy": { + "value": "azure" + }, + "omsAgentEnabled": { + "value": true + }, + "privateDNSZone": { + "value": "" + }, + "serviceCidr": { + "value": "10.10.200.0/24" + }, + "skuTier": { + "value": "Standard" + }, + "tags": { + "value": { + "Environment": "Non-Prod", + "hidden-title": "This is visible in the resource name", + "Role": "DeploymentValidation" + } + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/container-service/managed-cluster:' + +// Required parameters +param name = 'cswaf001' +param primaryAgentPoolProfile = [ + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } +] +// Non-required parameters +param agentPools = [ + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 60 + osDiskType: 'Ephemeral' + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 60 + osDiskType: 'Ephemeral' + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } +] +param autoUpgradeProfileUpgradeChannel = 'stable' +param diagnosticSettings = [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + logCategoriesAndGroups: [ + { + category: 'kube-apiserver' + } + { + category: 'kube-controller-manager' + } + { + category: 'kube-scheduler' + } + { + category: 'cluster-autoscaler' + } + ] + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } +] +param disableLocalAccounts = true +param dnsServiceIP = '10.10.200.10' +param enableAzureDefender = true +param enablePrivateCluster = true +param location = '' +param managedIdentities = { + userAssignedResourcesIds: [ + '' + ] +} +param monitoringWorkspaceId = '' +param networkPlugin = 'azure' +param networkPolicy = 'azure' +param omsAgentEnabled = true +param privateDNSZone = '' +param serviceCidr = '10.10.200.0/24' +param skuTier = 'Standard' +param tags = { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' +} +``` + +
+

+ +## Parameters + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-name) | string | Specifies the name of the AKS cluster. | +| [`primaryAgentPoolProfile`](#parameter-primaryagentpoolprofile) | array | Properties of the primary agent pool. | + +**Conditional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`aksServicePrincipalProfile`](#parameter-aksserviceprincipalprofile) | object | Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster. | +| [`appGatewayResourceId`](#parameter-appgatewayresourceid) | string | Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`aadProfileAdminGroupObjectIDs`](#parameter-aadprofileadmingroupobjectids) | array | Specifies the AAD group object IDs that will have admin role of the cluster. | +| [`aadProfileClientAppID`](#parameter-aadprofileclientappid) | string | The client AAD application ID. | +| [`aadProfileEnableAzureRBAC`](#parameter-aadprofileenableazurerbac) | bool | Specifies whether to enable Azure RBAC for Kubernetes authorization. | +| [`aadProfileManaged`](#parameter-aadprofilemanaged) | bool | Specifies whether to enable managed AAD integration. | +| [`aadProfileServerAppID`](#parameter-aadprofileserverappid) | string | The server AAD application ID. | +| [`aadProfileServerAppSecret`](#parameter-aadprofileserverappsecret) | string | The server AAD application secret. | +| [`aadProfileTenantId`](#parameter-aadprofiletenantid) | string | Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication. | +| [`aciConnectorLinuxEnabled`](#parameter-aciconnectorlinuxenabled) | bool | Specifies whether the aciConnectorLinux add-on is enabled or not. | +| [`adminUsername`](#parameter-adminusername) | string | Specifies the administrator username of Linux virtual machines. | +| [`agentPools`](#parameter-agentpools) | array | Define one or more secondary/additional agent pools. | +| [`authorizedIPRanges`](#parameter-authorizedipranges) | array | IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer. | +| [`autoScalerProfileBalanceSimilarNodeGroups`](#parameter-autoscalerprofilebalancesimilarnodegroups) | bool | Specifies the balance of similar node groups for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileExpander`](#parameter-autoscalerprofileexpander) | string | Specifies the expand strategy for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileMaxEmptyBulkDelete`](#parameter-autoscalerprofilemaxemptybulkdelete) | string | Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileMaxGracefulTerminationSec`](#parameter-autoscalerprofilemaxgracefulterminationsec) | string | Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileMaxNodeProvisionTime`](#parameter-autoscalerprofilemaxnodeprovisiontime) | string | Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported. | +| [`autoScalerProfileMaxTotalUnreadyPercentage`](#parameter-autoscalerprofilemaxtotalunreadypercentage) | string | Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0. | +| [`autoScalerProfileNewPodScaleUpDelay`](#parameter-autoscalerprofilenewpodscaleupdelay) | string | For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc). | +| [`autoScalerProfileOkTotalUnreadyCount`](#parameter-autoscalerprofileoktotalunreadycount) | string | Specifies the OK total unready count for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownDelayAfterAdd`](#parameter-autoscalerprofilescaledowndelayafteradd) | string | Specifies the scale down delay after add of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownDelayAfterDelete`](#parameter-autoscalerprofilescaledowndelayafterdelete) | string | Specifies the scale down delay after delete of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownDelayAfterFailure`](#parameter-autoscalerprofilescaledowndelayafterfailure) | string | Specifies scale down delay after failure of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownUnneededTime`](#parameter-autoscalerprofilescaledownunneededtime) | string | Specifies the scale down unneeded time of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownUnreadyTime`](#parameter-autoscalerprofilescaledownunreadytime) | string | Specifies the scale down unready time of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScanInterval`](#parameter-autoscalerprofilescaninterval) | string | Specifies the scan interval of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileSkipNodesWithLocalStorage`](#parameter-autoscalerprofileskipnodeswithlocalstorage) | bool | Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileSkipNodesWithSystemPods`](#parameter-autoscalerprofileskipnodeswithsystempods) | bool | Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileUtilizationThreshold`](#parameter-autoscalerprofileutilizationthreshold) | string | Specifies the utilization threshold of the auto-scaler of the AKS cluster. | +| [`autoUpgradeProfileUpgradeChannel`](#parameter-autoupgradeprofileupgradechannel) | string | Auto-upgrade channel on the AKS cluster. | +| [`azurePolicyEnabled`](#parameter-azurepolicyenabled) | bool | Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled. | +| [`azurePolicyVersion`](#parameter-azurepolicyversion) | string | Specifies the azure policy version to use. | +| [`backendPoolType`](#parameter-backendpooltype) | string | The type of the managed inbound Load Balancer BackendPool. | +| [`costAnalysisEnabled`](#parameter-costanalysisenabled) | bool | Specifies whether the cost analysis add-on is enabled or not. If Enabled `enableStorageProfileDiskCSIDriver` is set to true as it is needed. | +| [`customerManagedKey`](#parameter-customermanagedkey) | object | The customer managed key definition. | +| [`diagnosticSettings`](#parameter-diagnosticsettings) | array | The diagnostic settings of the service. | +| [`disableCustomMetrics`](#parameter-disablecustommetrics) | bool | Indicates whether custom metrics collection has to be disabled or not. If not specified the default is false. No custom metrics will be emitted if this field is false but the container insights enabled field is false. | +| [`disableLocalAccounts`](#parameter-disablelocalaccounts) | bool | If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled. | +| [`disablePrometheusMetricsScraping`](#parameter-disableprometheusmetricsscraping) | bool | Indicates whether prometheus metrics scraping is disabled or not. If not specified the default is false. No prometheus metrics will be emitted if this field is false but the container insights enabled field is false. | +| [`disableRunCommand`](#parameter-disableruncommand) | bool | Whether to disable run command for the cluster or not. | +| [`diskEncryptionSetResourceId`](#parameter-diskencryptionsetresourceid) | string | The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided. | +| [`dnsPrefix`](#parameter-dnsprefix) | string | Specifies the DNS prefix specified when creating the managed cluster. | +| [`dnsServiceIP`](#parameter-dnsserviceip) | string | Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. | +| [`dnsZoneResourceId`](#parameter-dnszoneresourceid) | string | Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`. | +| [`enableAzureDefender`](#parameter-enableazuredefender) | bool | Whether to enable Azure Defender. | +| [`enableAzureMonitorProfileMetrics`](#parameter-enableazuremonitorprofilemetrics) | bool | Whether the metric state of the kubenetes cluster is enabled. | +| [`enableContainerInsights`](#parameter-enablecontainerinsights) | bool | Indicates if Azure Monitor Container Insights Logs Addon is enabled. | +| [`enableDnsZoneContributorRoleAssignment`](#parameter-enablednszonecontributorroleassignment) | bool | Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided. | +| [`enableImageCleaner`](#parameter-enableimagecleaner) | bool | Whether to enable Image Cleaner for Kubernetes. | +| [`enableKeyvaultSecretsProvider`](#parameter-enablekeyvaultsecretsprovider) | bool | Specifies whether the KeyvaultSecretsProvider add-on is enabled or not. | +| [`enableOidcIssuerProfile`](#parameter-enableoidcissuerprofile) | bool | Whether the The OIDC issuer profile of the Managed Cluster is enabled. | +| [`enablePodSecurityPolicy`](#parameter-enablepodsecuritypolicy) | bool | Whether to enable Kubernetes pod security policy. Requires enabling the pod security policy feature flag on the subscription. | +| [`enablePrivateCluster`](#parameter-enableprivatecluster) | bool | Specifies whether to create the cluster as a private cluster or not. | +| [`enablePrivateClusterPublicFQDN`](#parameter-enableprivateclusterpublicfqdn) | bool | Whether to create additional public FQDN for private cluster or not. | +| [`enableRBAC`](#parameter-enablerbac) | bool | Whether to enable Kubernetes Role-Based Access Control. | +| [`enableSecretRotation`](#parameter-enablesecretrotation) | bool | Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. | +| [`enableStorageProfileBlobCSIDriver`](#parameter-enablestorageprofileblobcsidriver) | bool | Whether the AzureBlob CSI Driver for the storage profile is enabled. | +| [`enableStorageProfileDiskCSIDriver`](#parameter-enablestorageprofilediskcsidriver) | bool | Whether the AzureDisk CSI Driver for the storage profile is enabled. | +| [`enableStorageProfileFileCSIDriver`](#parameter-enablestorageprofilefilecsidriver) | bool | Whether the AzureFile CSI Driver for the storage profile is enabled. | +| [`enableStorageProfileSnapshotController`](#parameter-enablestorageprofilesnapshotcontroller) | bool | Whether the snapshot controller for the storage profile is enabled. | +| [`enableTelemetry`](#parameter-enabletelemetry) | bool | Enable/Disable usage telemetry for module. | +| [`enableWorkloadIdentity`](#parameter-enableworkloadidentity) | bool | Whether to enable Workload Identity. Requires OIDC issuer profile to be enabled. | +| [`fluxExtension`](#parameter-fluxextension) | object | Settings and configurations for the flux extension. | +| [`httpApplicationRoutingEnabled`](#parameter-httpapplicationroutingenabled) | bool | Specifies whether the httpApplicationRouting add-on is enabled or not. | +| [`httpProxyConfig`](#parameter-httpproxyconfig) | object | Configurations for provisioning the cluster with HTTP proxy servers. | +| [`identityProfile`](#parameter-identityprofile) | object | Identities associated with the cluster. | +| [`imageCleanerIntervalHours`](#parameter-imagecleanerintervalhours) | int | The interval in hours Image Cleaner will run. The maximum value is three months. | +| [`ingressApplicationGatewayEnabled`](#parameter-ingressapplicationgatewayenabled) | bool | Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. | +| [`kedaAddon`](#parameter-kedaaddon) | bool | Enables Kubernetes Event-driven Autoscaling (KEDA). | +| [`kubeDashboardEnabled`](#parameter-kubedashboardenabled) | bool | Specifies whether the kubeDashboard add-on is enabled or not. | +| [`kubernetesVersion`](#parameter-kubernetesversion) | string | Version of Kubernetes specified when creating the managed cluster. | +| [`loadBalancerSku`](#parameter-loadbalancersku) | string | Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools. | +| [`location`](#parameter-location) | string | Specifies the location of AKS cluster. It picks up Resource Group's location by default. | +| [`lock`](#parameter-lock) | object | The lock settings of the service. | +| [`maintenanceConfiguration`](#parameter-maintenanceconfiguration) | object | Whether or not to use AKS Automatic mode. | +| [`managedIdentities`](#parameter-managedidentities) | object | The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both. | +| [`managedOutboundIPCount`](#parameter-managedoutboundipcount) | int | Outbound IP Count for the Load balancer. | +| [`metricAnnotationsAllowList`](#parameter-metricannotationsallowlist) | string | A comma-separated list of Kubernetes cluster metrics annotations. | +| [`metricLabelsAllowlist`](#parameter-metriclabelsallowlist) | string | A comma-separated list of kubernetes cluster metrics labels. | +| [`monitoringWorkspaceId`](#parameter-monitoringworkspaceid) | string | Resource ID of the monitoring log analytics workspace. | +| [`networkDataplane`](#parameter-networkdataplane) | string | Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin. | +| [`networkPlugin`](#parameter-networkplugin) | string | Specifies the network plugin used for building Kubernetes network. | +| [`networkPluginMode`](#parameter-networkpluginmode) | string | Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin. | +| [`networkPolicy`](#parameter-networkpolicy) | string | Specifies the network policy used for building Kubernetes network. - calico or azure. | +| [`nodeResourceGroup`](#parameter-noderesourcegroup) | string | Name of the resource group containing agent pool nodes. | +| [`omsAgentEnabled`](#parameter-omsagentenabled) | bool | Specifies whether the OMS agent is enabled. | +| [`openServiceMeshEnabled`](#parameter-openservicemeshenabled) | bool | Specifies whether the openServiceMesh add-on is enabled or not. | +| [`outboundType`](#parameter-outboundtype) | string | Specifies outbound (egress) routing method. | +| [`podCidr`](#parameter-podcidr) | string | Specifies the CIDR notation IP range from which to assign pod IPs when kubenet is used. | +| [`podIdentityProfileAllowNetworkPluginKubenet`](#parameter-podidentityprofileallownetworkpluginkubenet) | bool | Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing. | +| [`podIdentityProfileEnable`](#parameter-podidentityprofileenable) | bool | Whether the pod identity addon is enabled. | +| [`podIdentityProfileUserAssignedIdentities`](#parameter-podidentityprofileuserassignedidentities) | array | The pod identities to use in the cluster. | +| [`podIdentityProfileUserAssignedIdentityExceptions`](#parameter-podidentityprofileuserassignedidentityexceptions) | array | The pod identity exceptions to allow. | +| [`privateDNSZone`](#parameter-privatednszone) | string | Private DNS Zone configuration. Set to 'system' and AKS will create a private DNS zone in the node resource group. Set to '' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone. | +| [`publicNetworkAccess`](#parameter-publicnetworkaccess) | string | Allow or deny public network access for AKS. | +| [`roleAssignments`](#parameter-roleassignments) | array | Array of role assignments to create. | +| [`serviceCidr`](#parameter-servicecidr) | string | A CIDR notation IP range from which to assign service cluster IPs. It must not overlap with any Subnet IP ranges. | +| [`skuTier`](#parameter-skutier) | string | Tier of a managed cluster SKU. | +| [`sshPublicKey`](#parameter-sshpublickey) | string | Specifies the SSH RSA public key string for the Linux nodes. | +| [`supportPlan`](#parameter-supportplan) | string | The support plan for the Managed Cluster. | +| [`syslogPort`](#parameter-syslogport) | int | The syslog host port. If not specified, the default port is 28330. | +| [`tags`](#parameter-tags) | object | Tags of the resource. | +| [`vpaAddon`](#parameter-vpaaddon) | bool | Whether to enable VPA add-on in cluster. Default value is false. | +| [`webApplicationRoutingEnabled`](#parameter-webapplicationroutingenabled) | bool | Specifies whether the webApplicationRoutingEnabled add-on is enabled or not. | + +### Parameter: `name` + +Specifies the name of the AKS cluster. + +- Required: Yes +- Type: string + +### Parameter: `primaryAgentPoolProfile` + +Properties of the primary agent pool. + +- Required: Yes +- Type: array + +### Parameter: `aksServicePrincipalProfile` + +Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster. + +- Required: No +- Type: object + +### Parameter: `appGatewayResourceId` + +Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`. + +- Required: No +- Type: string + +### Parameter: `aadProfileAdminGroupObjectIDs` + +Specifies the AAD group object IDs that will have admin role of the cluster. + +- Required: No +- Type: array + +### Parameter: `aadProfileClientAppID` + +The client AAD application ID. + +- Required: No +- Type: string + +### Parameter: `aadProfileEnableAzureRBAC` + +Specifies whether to enable Azure RBAC for Kubernetes authorization. + +- Required: No +- Type: bool +- Default: `[parameters('enableRBAC')]` + +### Parameter: `aadProfileManaged` + +Specifies whether to enable managed AAD integration. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `aadProfileServerAppID` + +The server AAD application ID. + +- Required: No +- Type: string + +### Parameter: `aadProfileServerAppSecret` + +The server AAD application secret. + +- Required: No +- Type: string + +### Parameter: `aadProfileTenantId` + +Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication. + +- Required: No +- Type: string +- Default: `[subscription().tenantId]` + +### Parameter: `aciConnectorLinuxEnabled` + +Specifies whether the aciConnectorLinux add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `adminUsername` + +Specifies the administrator username of Linux virtual machines. + +- Required: No +- Type: string +- Default: `'azureuser'` + +### Parameter: `agentPools` + +Define one or more secondary/additional agent pools. + +- Required: No +- Type: array + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-agentpoolsname) | string | The name of the agent pool. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`availabilityZones`](#parameter-agentpoolsavailabilityzones) | array | The availability zones of the agent pool. | +| [`count`](#parameter-agentpoolscount) | int | The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`enableAutoScaling`](#parameter-agentpoolsenableautoscaling) | bool | Whether to enable auto-scaling for the agent pool. | +| [`enableDefaultTelemetry`](#parameter-agentpoolsenabledefaulttelemetry) | bool | The enable default telemetry of the agent pool. | +| [`enableEncryptionAtHost`](#parameter-agentpoolsenableencryptionathost) | bool | Whether to enable encryption at host for the agent pool. | +| [`enableFIPS`](#parameter-agentpoolsenablefips) | bool | Whether to enable FIPS for the agent pool. | +| [`enableNodePublicIP`](#parameter-agentpoolsenablenodepublicip) | bool | Whether to enable node public IP for the agent pool. | +| [`enableUltraSSD`](#parameter-agentpoolsenableultrassd) | bool | Whether to enable Ultra SSD for the agent pool. | +| [`gpuInstanceProfile`](#parameter-agentpoolsgpuinstanceprofile) | string | The GPU instance profile of the agent pool. | +| [`kubeletDiskType`](#parameter-agentpoolskubeletdisktype) | string | The kubelet disk type of the agent pool. | +| [`maxCount`](#parameter-agentpoolsmaxcount) | int | The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`maxPods`](#parameter-agentpoolsmaxpods) | int | The maximum number of pods that can run on a node. | +| [`maxSurge`](#parameter-agentpoolsmaxsurge) | string | The maximum number of nodes that can be created during an upgrade. | +| [`minCount`](#parameter-agentpoolsmincount) | int | The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`minPods`](#parameter-agentpoolsminpods) | int | The minimum number of pods that can run on a node. | +| [`mode`](#parameter-agentpoolsmode) | string | The mode of the agent pool. | +| [`nodeLabels`](#parameter-agentpoolsnodelabels) | object | The node labels of the agent pool. | +| [`nodePublicIpPrefixId`](#parameter-agentpoolsnodepublicipprefixid) | string | The node public IP prefix ID of the agent pool. | +| [`nodeTaints`](#parameter-agentpoolsnodetaints) | array | The node taints of the agent pool. | +| [`orchestratorVersion`](#parameter-agentpoolsorchestratorversion) | string | The Kubernetes version of the agent pool. | +| [`osDiskSizeGB`](#parameter-agentpoolsosdisksizegb) | int | The OS disk size in GB of the agent pool. | +| [`osDiskType`](#parameter-agentpoolsosdisktype) | string | The OS disk type of the agent pool. | +| [`osSku`](#parameter-agentpoolsossku) | string | The OS SKU of the agent pool. | +| [`osType`](#parameter-agentpoolsostype) | string | The OS type of the agent pool. | +| [`podSubnetId`](#parameter-agentpoolspodsubnetid) | string | The pod subnet ID of the agent pool. | +| [`proximityPlacementGroupResourceId`](#parameter-agentpoolsproximityplacementgroupresourceid) | string | The proximity placement group resource ID of the agent pool. | +| [`scaleDownMode`](#parameter-agentpoolsscaledownmode) | string | The scale down mode of the agent pool. | +| [`scaleSetEvictionPolicy`](#parameter-agentpoolsscalesetevictionpolicy) | string | The scale set eviction policy of the agent pool. | +| [`scaleSetPriority`](#parameter-agentpoolsscalesetpriority) | string | The scale set priority of the agent pool. | +| [`sourceResourceId`](#parameter-agentpoolssourceresourceid) | string | The source resource ID to create the agent pool from. | +| [`spotMaxPrice`](#parameter-agentpoolsspotmaxprice) | int | The spot max price of the agent pool. | +| [`tags`](#parameter-agentpoolstags) | object | The tags of the agent pool. | +| [`type`](#parameter-agentpoolstype) | string | The type of the agent pool. | +| [`vmSize`](#parameter-agentpoolsvmsize) | string | The VM size of the agent pool. | +| [`vnetSubnetID`](#parameter-agentpoolsvnetsubnetid) | string | The VNet subnet ID of the agent pool. | +| [`workloadRuntime`](#parameter-agentpoolsworkloadruntime) | string | The workload runtime of the agent pool. | + +### Parameter: `agentPools.name` + +The name of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.availabilityZones` + +The availability zones of the agent pool. + +- Required: No +- Type: array + +### Parameter: `agentPools.count` + +The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `agentPools.enableAutoScaling` + +Whether to enable auto-scaling for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableDefaultTelemetry` + +The enable default telemetry of the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableEncryptionAtHost` + +Whether to enable encryption at host for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableFIPS` + +Whether to enable FIPS for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableNodePublicIP` + +Whether to enable node public IP for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableUltraSSD` + +Whether to enable Ultra SSD for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.gpuInstanceProfile` + +The GPU instance profile of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'MIG1g' + 'MIG2g' + 'MIG3g' + 'MIG4g' + 'MIG7g' + ] + ``` + +### Parameter: `agentPools.kubeletDiskType` + +The kubelet disk type of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.maxCount` + +The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `agentPools.maxPods` + +The maximum number of pods that can run on a node. + +- Required: No +- Type: int + +### Parameter: `agentPools.maxSurge` + +The maximum number of nodes that can be created during an upgrade. + +- Required: No +- Type: string + +### Parameter: `agentPools.minCount` + +The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `agentPools.minPods` + +The minimum number of pods that can run on a node. + +- Required: No +- Type: int + +### Parameter: `agentPools.mode` + +The mode of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'System' + 'User' + ] + ``` + +### Parameter: `agentPools.nodeLabels` + +The node labels of the agent pool. + +- Required: No +- Type: object + +### Parameter: `agentPools.nodePublicIpPrefixId` + +The node public IP prefix ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.nodeTaints` + +The node taints of the agent pool. + +- Required: No +- Type: array + +### Parameter: `agentPools.orchestratorVersion` + +The Kubernetes version of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.osDiskSizeGB` + +The OS disk size in GB of the agent pool. + +- Required: No +- Type: int + +### Parameter: `agentPools.osDiskType` + +The OS disk type of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.osSku` + +The OS SKU of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.osType` + +The OS type of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Linux' + 'Windows' + ] + ``` + +### Parameter: `agentPools.podSubnetId` + +The pod subnet ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.proximityPlacementGroupResourceId` + +The proximity placement group resource ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.scaleDownMode` + +The scale down mode of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `agentPools.scaleSetEvictionPolicy` + +The scale set eviction policy of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `agentPools.scaleSetPriority` + +The scale set priority of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Low' + 'Regular' + 'Spot' + ] + ``` + +### Parameter: `agentPools.sourceResourceId` + +The source resource ID to create the agent pool from. + +- Required: No +- Type: string + +### Parameter: `agentPools.spotMaxPrice` + +The spot max price of the agent pool. + +- Required: No +- Type: int + +### Parameter: `agentPools.tags` + +The tags of the agent pool. + +- Required: No +- Type: object + +### Parameter: `agentPools.type` + +The type of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'AvailabilitySet' + 'VirtualMachineScaleSets' + ] + ``` + +### Parameter: `agentPools.vmSize` + +The VM size of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.vnetSubnetID` + +The VNet subnet ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.workloadRuntime` + +The workload runtime of the agent pool. + +- Required: No +- Type: string + +### Parameter: `authorizedIPRanges` + +IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer. + +- Required: No +- Type: array + +### Parameter: `autoScalerProfileBalanceSimilarNodeGroups` + +Specifies the balance of similar node groups for the auto-scaler of the AKS cluster. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `autoScalerProfileExpander` + +Specifies the expand strategy for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'random'` +- Allowed: + ```Bicep + [ + 'least-waste' + 'most-pods' + 'priority' + 'random' + ] + ``` + +### Parameter: `autoScalerProfileMaxEmptyBulkDelete` + +Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'10'` + +### Parameter: `autoScalerProfileMaxGracefulTerminationSec` + +Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'600'` + +### Parameter: `autoScalerProfileMaxNodeProvisionTime` + +Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported. + +- Required: No +- Type: string +- Default: `'15m'` + +### Parameter: `autoScalerProfileMaxTotalUnreadyPercentage` + +Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0. + +- Required: No +- Type: string +- Default: `'45'` + +### Parameter: `autoScalerProfileNewPodScaleUpDelay` + +For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc). + +- Required: No +- Type: string +- Default: `'0s'` + +### Parameter: `autoScalerProfileOkTotalUnreadyCount` + +Specifies the OK total unready count for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'3'` + +### Parameter: `autoScalerProfileScaleDownDelayAfterAdd` + +Specifies the scale down delay after add of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'10m'` + +### Parameter: `autoScalerProfileScaleDownDelayAfterDelete` + +Specifies the scale down delay after delete of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'20s'` + +### Parameter: `autoScalerProfileScaleDownDelayAfterFailure` + +Specifies scale down delay after failure of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'3m'` + +### Parameter: `autoScalerProfileScaleDownUnneededTime` + +Specifies the scale down unneeded time of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'10m'` + +### Parameter: `autoScalerProfileScaleDownUnreadyTime` + +Specifies the scale down unready time of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'20m'` + +### Parameter: `autoScalerProfileScanInterval` + +Specifies the scan interval of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'10s'` + +### Parameter: `autoScalerProfileSkipNodesWithLocalStorage` + +Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `autoScalerProfileSkipNodesWithSystemPods` + +Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `autoScalerProfileUtilizationThreshold` + +Specifies the utilization threshold of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'0.5'` + +### Parameter: `autoUpgradeProfileUpgradeChannel` + +Auto-upgrade channel on the AKS cluster. + +- Required: No +- Type: string +- Default: `'stable'` +- Allowed: + ```Bicep + [ + 'node-image' + 'none' + 'patch' + 'rapid' + 'stable' + ] + ``` + +### Parameter: `azurePolicyEnabled` + +Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `azurePolicyVersion` + +Specifies the azure policy version to use. + +- Required: No +- Type: string +- Default: `'v2'` + +### Parameter: `backendPoolType` + +The type of the managed inbound Load Balancer BackendPool. + +- Required: No +- Type: string +- Default: `'NodeIPConfiguration'` +- Allowed: + ```Bicep + [ + 'NodeIP' + 'NodeIPConfiguration' + ] + ``` + +### Parameter: `costAnalysisEnabled` + +Specifies whether the cost analysis add-on is enabled or not. If Enabled `enableStorageProfileDiskCSIDriver` is set to true as it is needed. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `customerManagedKey` + +The customer managed key definition. + +- Required: No +- Type: object + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`keyName`](#parameter-customermanagedkeykeyname) | string | The name of the customer managed key to use for encryption. | +| [`keyVaultNetworkAccess`](#parameter-customermanagedkeykeyvaultnetworkaccess) | string | Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public. | +| [`keyVaultResourceId`](#parameter-customermanagedkeykeyvaultresourceid) | string | The resource ID of a key vault to reference a customer managed key for encryption from. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`keyVersion`](#parameter-customermanagedkeykeyversion) | string | The version of the customer managed key to reference for encryption. If not provided, using 'latest'. | + +### Parameter: `customerManagedKey.keyName` + +The name of the customer managed key to use for encryption. + +- Required: Yes +- Type: string + +### Parameter: `customerManagedKey.keyVaultNetworkAccess` + +Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public. + +- Required: Yes +- Type: string +- Allowed: + ```Bicep + [ + 'Private' + 'Public' + ] + ``` + +### Parameter: `customerManagedKey.keyVaultResourceId` + +The resource ID of a key vault to reference a customer managed key for encryption from. + +- Required: Yes +- Type: string + +### Parameter: `customerManagedKey.keyVersion` + +The version of the customer managed key to reference for encryption. If not provided, using 'latest'. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings` + +The diagnostic settings of the service. + +- Required: No +- Type: array + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`eventHubAuthorizationRuleResourceId`](#parameter-diagnosticsettingseventhubauthorizationruleresourceid) | string | Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to. | +| [`eventHubName`](#parameter-diagnosticsettingseventhubname) | string | Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. | +| [`logAnalyticsDestinationType`](#parameter-diagnosticsettingsloganalyticsdestinationtype) | string | A string indicating whether the export to Log Analytics should use the default destination type, i.e. AzureDiagnostics, or use a destination type. | +| [`logCategoriesAndGroups`](#parameter-diagnosticsettingslogcategoriesandgroups) | array | The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to `[]` to disable log collection. | +| [`marketplacePartnerResourceId`](#parameter-diagnosticsettingsmarketplacepartnerresourceid) | string | The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs. | +| [`metricCategories`](#parameter-diagnosticsettingsmetriccategories) | array | The name of metrics that will be streamed. "allMetrics" includes all possible metrics for the resource. Set to `[]` to disable metric collection. | +| [`name`](#parameter-diagnosticsettingsname) | string | The name of diagnostic setting. | +| [`storageAccountResourceId`](#parameter-diagnosticsettingsstorageaccountresourceid) | string | Resource ID of the diagnostic storage account. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. | +| [`workspaceResourceId`](#parameter-diagnosticsettingsworkspaceresourceid) | string | Resource ID of the diagnostic log analytics workspace. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. | + +### Parameter: `diagnosticSettings.eventHubAuthorizationRuleResourceId` + +Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.eventHubName` + +Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.logAnalyticsDestinationType` + +A string indicating whether the export to Log Analytics should use the default destination type, i.e. AzureDiagnostics, or use a destination type. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'AzureDiagnostics' + 'Dedicated' + ] + ``` + +### Parameter: `diagnosticSettings.logCategoriesAndGroups` + +The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to `[]` to disable log collection. + +- Required: No +- Type: array + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`category`](#parameter-diagnosticsettingslogcategoriesandgroupscategory) | string | Name of a Diagnostic Log category for a resource type this setting is applied to. Set the specific logs to collect here. | +| [`categoryGroup`](#parameter-diagnosticsettingslogcategoriesandgroupscategorygroup) | string | Name of a Diagnostic Log category group for a resource type this setting is applied to. Set to `allLogs` to collect all logs. | +| [`enabled`](#parameter-diagnosticsettingslogcategoriesandgroupsenabled) | bool | Enable or disable the category explicitly. Default is `true`. | + +### Parameter: `diagnosticSettings.logCategoriesAndGroups.category` + +Name of a Diagnostic Log category for a resource type this setting is applied to. Set the specific logs to collect here. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.logCategoriesAndGroups.categoryGroup` + +Name of a Diagnostic Log category group for a resource type this setting is applied to. Set to `allLogs` to collect all logs. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.logCategoriesAndGroups.enabled` + +Enable or disable the category explicitly. Default is `true`. + +- Required: No +- Type: bool + +### Parameter: `diagnosticSettings.marketplacePartnerResourceId` + +The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.metricCategories` + +The name of metrics that will be streamed. "allMetrics" includes all possible metrics for the resource. Set to `[]` to disable metric collection. + +- Required: No +- Type: array + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`category`](#parameter-diagnosticsettingsmetriccategoriescategory) | string | Name of a Diagnostic Metric category for a resource type this setting is applied to. Set to `AllMetrics` to collect all metrics. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`enabled`](#parameter-diagnosticsettingsmetriccategoriesenabled) | bool | Enable or disable the category explicitly. Default is `true`. | + +### Parameter: `diagnosticSettings.metricCategories.category` + +Name of a Diagnostic Metric category for a resource type this setting is applied to. Set to `AllMetrics` to collect all metrics. + +- Required: Yes +- Type: string + +### Parameter: `diagnosticSettings.metricCategories.enabled` + +Enable or disable the category explicitly. Default is `true`. + +- Required: No +- Type: bool + +### Parameter: `diagnosticSettings.name` + +The name of diagnostic setting. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.storageAccountResourceId` + +Resource ID of the diagnostic storage account. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.workspaceResourceId` + +Resource ID of the diagnostic log analytics workspace. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. + +- Required: No +- Type: string + +### Parameter: `disableCustomMetrics` + +Indicates whether custom metrics collection has to be disabled or not. If not specified the default is false. No custom metrics will be emitted if this field is false but the container insights enabled field is false. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `disableLocalAccounts` + +If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `disablePrometheusMetricsScraping` + +Indicates whether prometheus metrics scraping is disabled or not. If not specified the default is false. No prometheus metrics will be emitted if this field is false but the container insights enabled field is false. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `disableRunCommand` + +Whether to disable run command for the cluster or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `diskEncryptionSetResourceId` + +The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided. + +- Required: No +- Type: string + +### Parameter: `dnsPrefix` + +Specifies the DNS prefix specified when creating the managed cluster. + +- Required: No +- Type: string +- Default: `[parameters('name')]` + +### Parameter: `dnsServiceIP` + +Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. + +- Required: No +- Type: string + +### Parameter: `dnsZoneResourceId` + +Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`. + +- Required: No +- Type: string + +### Parameter: `enableAzureDefender` + +Whether to enable Azure Defender. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableAzureMonitorProfileMetrics` + +Whether the metric state of the kubenetes cluster is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableContainerInsights` + +Indicates if Azure Monitor Container Insights Logs Addon is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableDnsZoneContributorRoleAssignment` + +Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `enableImageCleaner` + +Whether to enable Image Cleaner for Kubernetes. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableKeyvaultSecretsProvider` + +Specifies whether the KeyvaultSecretsProvider add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableOidcIssuerProfile` + +Whether the The OIDC issuer profile of the Managed Cluster is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enablePodSecurityPolicy` + +Whether to enable Kubernetes pod security policy. Requires enabling the pod security policy feature flag on the subscription. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enablePrivateCluster` + +Specifies whether to create the cluster as a private cluster or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enablePrivateClusterPublicFQDN` + +Whether to create additional public FQDN for private cluster or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableRBAC` + +Whether to enable Kubernetes Role-Based Access Control. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `enableSecretRotation` + +Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableStorageProfileBlobCSIDriver` + +Whether the AzureBlob CSI Driver for the storage profile is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableStorageProfileDiskCSIDriver` + +Whether the AzureDisk CSI Driver for the storage profile is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableStorageProfileFileCSIDriver` + +Whether the AzureFile CSI Driver for the storage profile is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableStorageProfileSnapshotController` + +Whether the snapshot controller for the storage profile is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableTelemetry` + +Enable/Disable usage telemetry for module. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `enableWorkloadIdentity` + +Whether to enable Workload Identity. Requires OIDC issuer profile to be enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `fluxExtension` + +Settings and configurations for the flux extension. + +- Required: No +- Type: object + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-fluxextensionname) | string | The name of the extension. | +| [`releaseTrain`](#parameter-fluxextensionreleasetrain) | string | The release train of the extension. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`configurationProtectedSettings`](#parameter-fluxextensionconfigurationprotectedsettings) | object | The configuration protected settings of the extension. | +| [`configurations`](#parameter-fluxextensionconfigurations) | array | The flux configurations of the extension. | +| [`configurationSettings`](#parameter-fluxextensionconfigurationsettings) | object | The configuration settings of the extension. | +| [`releaseNamespace`](#parameter-fluxextensionreleasenamespace) | string | Namespace where the extension Release must be placed. | +| [`targetNamespace`](#parameter-fluxextensiontargetnamespace) | string | Namespace where the extension will be created for an Namespace scoped extension. | +| [`version`](#parameter-fluxextensionversion) | string | The version of the extension. | + +### Parameter: `fluxExtension.name` + +The name of the extension. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.releaseTrain` + +The release train of the extension. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.configurationProtectedSettings` + +The configuration protected settings of the extension. + +- Required: No +- Type: object + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`sshPrivateKey`](#parameter-fluxextensionconfigurationprotectedsettingssshprivatekey) | string | The SSH private key to use for Git authentication. | + +### Parameter: `fluxExtension.configurationProtectedSettings.sshPrivateKey` + +The SSH private key to use for Git authentication. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.configurations` + +The flux configurations of the extension. + +- Required: No +- Type: array + +### Parameter: `fluxExtension.configurationSettings` + +The configuration settings of the extension. + +- Required: No +- Type: object + +### Parameter: `fluxExtension.releaseNamespace` + +Namespace where the extension Release must be placed. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.targetNamespace` + +Namespace where the extension will be created for an Namespace scoped extension. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.version` + +The version of the extension. + +- Required: No +- Type: string + +### Parameter: `httpApplicationRoutingEnabled` + +Specifies whether the httpApplicationRouting add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `httpProxyConfig` + +Configurations for provisioning the cluster with HTTP proxy servers. + +- Required: No +- Type: object + +### Parameter: `identityProfile` + +Identities associated with the cluster. + +- Required: No +- Type: object + +### Parameter: `imageCleanerIntervalHours` + +The interval in hours Image Cleaner will run. The maximum value is three months. + +- Required: No +- Type: int +- Default: `24` + +### Parameter: `ingressApplicationGatewayEnabled` + +Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `kedaAddon` + +Enables Kubernetes Event-driven Autoscaling (KEDA). + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `kubeDashboardEnabled` + +Specifies whether the kubeDashboard add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `kubernetesVersion` + +Version of Kubernetes specified when creating the managed cluster. + +- Required: No +- Type: string + +### Parameter: `loadBalancerSku` + +Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools. + +- Required: No +- Type: string +- Default: `'standard'` +- Allowed: + ```Bicep + [ + 'basic' + 'standard' + ] + ``` + +### Parameter: `location` + +Specifies the location of AKS cluster. It picks up Resource Group's location by default. + +- Required: No +- Type: string +- Default: `[resourceGroup().location]` + +### Parameter: `lock` + +The lock settings of the service. + +- Required: No +- Type: object + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`kind`](#parameter-lockkind) | string | Specify the type of lock. | +| [`name`](#parameter-lockname) | string | Specify the name of lock. | + +### Parameter: `lock.kind` + +Specify the type of lock. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'CanNotDelete' + 'None' + 'ReadOnly' + ] + ``` + +### Parameter: `lock.name` + +Specify the name of lock. + +- Required: No +- Type: string + +### Parameter: `maintenanceConfiguration` + +Whether or not to use AKS Automatic mode. + +- Required: No +- Type: object + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`maintenanceWindow`](#parameter-maintenanceconfigurationmaintenancewindow) | object | Maintenance window for the maintenance configuration. | + +### Parameter: `maintenanceConfiguration.maintenanceWindow` + +Maintenance window for the maintenance configuration. + +- Required: Yes +- Type: object + +### Parameter: `managedIdentities` + +The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both. + +- Required: No +- Type: object + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`systemAssigned`](#parameter-managedidentitiessystemassigned) | bool | Enables system assigned managed identity on the resource. | +| [`userAssignedResourcesIds`](#parameter-managedidentitiesuserassignedresourcesids) | array | The resource ID(s) to assign to the resource. | + +### Parameter: `managedIdentities.systemAssigned` + +Enables system assigned managed identity on the resource. + +- Required: No +- Type: bool + +### Parameter: `managedIdentities.userAssignedResourcesIds` + +The resource ID(s) to assign to the resource. + +- Required: No +- Type: array + +### Parameter: `managedOutboundIPCount` + +Outbound IP Count for the Load balancer. + +- Required: No +- Type: int +- Default: `0` + +### Parameter: `metricAnnotationsAllowList` + +A comma-separated list of Kubernetes cluster metrics annotations. + +- Required: No +- Type: string +- Default: `''` + +### Parameter: `metricLabelsAllowlist` + +A comma-separated list of kubernetes cluster metrics labels. + +- Required: No +- Type: string +- Default: `''` + +### Parameter: `monitoringWorkspaceId` + +Resource ID of the monitoring log analytics workspace. + +- Required: No +- Type: string + +### Parameter: `networkDataplane` + +Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'azure' + 'cilium' + ] + ``` + +### Parameter: `networkPlugin` + +Specifies the network plugin used for building Kubernetes network. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'azure' + 'kubenet' + ] + ``` + +### Parameter: `networkPluginMode` + +Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'overlay' + ] + ``` + +### Parameter: `networkPolicy` + +Specifies the network policy used for building Kubernetes network. - calico or azure. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'azure' + 'calico' + ] + ``` + +### Parameter: `nodeResourceGroup` + +Name of the resource group containing agent pool nodes. + +- Required: No +- Type: string +- Default: `[format('{0}_aks_{1}_nodes', resourceGroup().name, parameters('name'))]` + +### Parameter: `omsAgentEnabled` + +Specifies whether the OMS agent is enabled. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `openServiceMeshEnabled` + +Specifies whether the openServiceMesh add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `outboundType` + +Specifies outbound (egress) routing method. + +- Required: No +- Type: string +- Default: `'loadBalancer'` +- Allowed: + ```Bicep + [ + 'loadBalancer' + 'managedNATGateway' + 'userAssignedNATGateway' + 'userDefinedRouting' + ] + ``` + +### Parameter: `podCidr` + +Specifies the CIDR notation IP range from which to assign pod IPs when kubenet is used. + +- Required: No +- Type: string + +### Parameter: `podIdentityProfileAllowNetworkPluginKubenet` + +Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `podIdentityProfileEnable` + +Whether the pod identity addon is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `podIdentityProfileUserAssignedIdentities` + +The pod identities to use in the cluster. + +- Required: No +- Type: array + +### Parameter: `podIdentityProfileUserAssignedIdentityExceptions` + +The pod identity exceptions to allow. + +- Required: No +- Type: array + +### Parameter: `privateDNSZone` + +Private DNS Zone configuration. Set to 'system' and AKS will create a private DNS zone in the node resource group. Set to '' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone. + +- Required: No +- Type: string + +### Parameter: `publicNetworkAccess` + +Allow or deny public network access for AKS. + +- Required: No +- Type: string +- Default: `'Disabled'` +- Allowed: + ```Bicep + [ + 'Disabled' + 'Enabled' + 'SecuredByPerimeter' + ] + ``` + +### Parameter: `roleAssignments` + +Array of role assignments to create. + +- Required: No +- Type: array +- Roles configurable by name: + - `'Azure Kubernetes Fleet Manager Contributor Role'` + - `'Azure Kubernetes Fleet Manager RBAC Admin'` + - `'Azure Kubernetes Fleet Manager RBAC Cluster Admin'` + - `'Azure Kubernetes Fleet Manager RBAC Reader'` + - `'Azure Kubernetes Fleet Manager RBAC Writer'` + - `'Azure Kubernetes Service Cluster Admin Role'` + - `'Azure Kubernetes Service Cluster Monitoring User'` + - `'Azure Kubernetes Service Cluster User Role'` + - `'Azure Kubernetes Service Contributor Role'` + - `'Azure Kubernetes Service RBAC Admin'` + - `'Azure Kubernetes Service RBAC Cluster Admin'` + - `'Azure Kubernetes Service RBAC Reader'` + - `'Azure Kubernetes Service RBAC Writer'` + - `'Contributor'` + - `'Kubernetes Agentless Operator'` + - `'Owner'` + - `'Reader'` + - `'Role Based Access Control Administrator (Preview)'` + - `'User Access Administrator'` + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`principalId`](#parameter-roleassignmentsprincipalid) | string | The principal ID of the principal (user/group/identity) to assign the role to. | +| [`roleDefinitionIdOrName`](#parameter-roleassignmentsroledefinitionidorname) | string | The role to assign. You can provide either the display name of the role definition, the role definition GUID, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`condition`](#parameter-roleassignmentscondition) | string | The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase "foo_storage_container". | +| [`conditionVersion`](#parameter-roleassignmentsconditionversion) | string | Version of the condition. | +| [`delegatedManagedIdentityResourceId`](#parameter-roleassignmentsdelegatedmanagedidentityresourceid) | string | The Resource Id of the delegated managed identity resource. | +| [`description`](#parameter-roleassignmentsdescription) | string | The description of the role assignment. | +| [`name`](#parameter-roleassignmentsname) | string | The name (as GUID) of the role assignment. If not provided, a GUID will be generated. | +| [`principalType`](#parameter-roleassignmentsprincipaltype) | string | The principal type of the assigned principal ID. | + +### Parameter: `roleAssignments.principalId` + +The principal ID of the principal (user/group/identity) to assign the role to. + +- Required: Yes +- Type: string + +### Parameter: `roleAssignments.roleDefinitionIdOrName` + +The role to assign. You can provide either the display name of the role definition, the role definition GUID, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'. + +- Required: Yes +- Type: string + +### Parameter: `roleAssignments.condition` + +The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase "foo_storage_container". + +- Required: No +- Type: string + +### Parameter: `roleAssignments.conditionVersion` + +Version of the condition. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + '2.0' + ] + ``` + +### Parameter: `roleAssignments.delegatedManagedIdentityResourceId` + +The Resource Id of the delegated managed identity resource. + +- Required: No +- Type: string + +### Parameter: `roleAssignments.description` + +The description of the role assignment. + +- Required: No +- Type: string + +### Parameter: `roleAssignments.name` + +The name (as GUID) of the role assignment. If not provided, a GUID will be generated. + +- Required: No +- Type: string + +### Parameter: `roleAssignments.principalType` + +The principal type of the assigned principal ID. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Device' + 'ForeignGroup' + 'Group' + 'ServicePrincipal' + 'User' + ] + ``` + +### Parameter: `serviceCidr` + +A CIDR notation IP range from which to assign service cluster IPs. It must not overlap with any Subnet IP ranges. + +- Required: No +- Type: string + +### Parameter: `skuTier` + +Tier of a managed cluster SKU. + +- Required: No +- Type: string +- Default: `'Standard'` +- Allowed: + ```Bicep + [ + 'Free' + 'Premium' + 'Standard' + ] + ``` + +### Parameter: `sshPublicKey` + +Specifies the SSH RSA public key string for the Linux nodes. + +- Required: No +- Type: string + +### Parameter: `supportPlan` + +The support plan for the Managed Cluster. + +- Required: No +- Type: string +- Default: `'KubernetesOfficial'` +- Allowed: + ```Bicep + [ + 'AKSLongTermSupport' + 'KubernetesOfficial' + ] + ``` + +### Parameter: `syslogPort` + +The syslog host port. If not specified, the default port is 28330. + +- Required: No +- Type: int +- Default: `28330` + +### Parameter: `tags` + +Tags of the resource. + +- Required: No +- Type: object + +### Parameter: `vpaAddon` + +Whether to enable VPA add-on in cluster. Default value is false. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `webApplicationRoutingEnabled` + +Specifies whether the webApplicationRoutingEnabled add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +## Outputs + +| Output | Type | Description | +| :-- | :-- | :-- | +| `addonProfiles` | object | The addonProfiles of the Kubernetes cluster. | +| `controlPlaneFQDN` | string | The control plane FQDN of the managed cluster. | +| `ingressApplicationGatewayIdentityObjectId` | string | The Object ID of Application Gateway Ingress Controller (AGIC) identity. | +| `keyvaultIdentityClientId` | string | The Client ID of the Key Vault Secrets Provider identity. | +| `keyvaultIdentityObjectId` | string | The Object ID of the Key Vault Secrets Provider identity. | +| `kubeletIdentityClientId` | string | The Client ID of the AKS identity. | +| `kubeletIdentityObjectId` | string | The Object ID of the AKS identity. | +| `kubeletIdentityResourceId` | string | The Resource ID of the AKS identity. | +| `location` | string | The location the resource was deployed into. | +| `name` | string | The name of the managed cluster. | +| `oidcIssuerUrl` | string | The OIDC token issuer URL. | +| `omsagentIdentityObjectId` | string | The Object ID of the OMS agent identity. | +| `resourceGroupName` | string | The resource group the managed cluster was deployed into. | +| `resourceId` | string | The resource ID of the managed cluster. | +| `systemAssignedMIPrincipalId` | string | The principal ID of the system assigned identity. | +| `webAppRoutingIdentityObjectId` | string | The Object ID of Web Application Routing. | + +## Cross-referenced modules + +This section gives you an overview of all local-referenced module files (i.e., other modules that are referenced in this module) and all remote-referenced files (i.e., Bicep modules that are referenced from a Bicep Registry or Template Specs). + +| Reference | Type | +| :-- | :-- | +| `br/public:avm/res/kubernetes-configuration/extension:0.2.0` | Remote reference | + +## Data Collection + +The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the [repository](https://aka.ms/avm/telemetry). There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft’s privacy statement. Our privacy statement is located at . You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. diff --git a/bicep/modules/managed-cluster/agent-pool/README.md b/bicep/modules/managed-cluster/agent-pool/README.md new file mode 100644 index 00000000..b83fb64a --- /dev/null +++ b/bicep/modules/managed-cluster/agent-pool/README.md @@ -0,0 +1,393 @@ +# Azure Kubernetes Service (AKS) Managed Cluster Agent Pools `[Microsoft.ContainerService/managedClusters/agentPools]` + +This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool. + +## Navigation + +- [Resource Types](#Resource-Types) +- [Parameters](#Parameters) +- [Outputs](#Outputs) + +## Resource Types + +| Resource Type | API Version | +| :-- | :-- | +| `Microsoft.ContainerService/managedClusters/agentPools` | [2023-07-02-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2023-07-02-preview/managedClusters/agentPools) | + +## Parameters + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-name) | string | Name of the agent pool. | + +**Conditional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`managedClusterName`](#parameter-managedclustername) | string | The name of the parent managed cluster. Required if the template is used in a standalone deployment. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`availabilityZones`](#parameter-availabilityzones) | array | The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets". | +| [`count`](#parameter-count) | int | Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1. | +| [`enableAutoScaling`](#parameter-enableautoscaling) | bool | Whether to enable auto-scaler. | +| [`enableEncryptionAtHost`](#parameter-enableencryptionathost) | bool | This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled. | +| [`enableFIPS`](#parameter-enablefips) | bool | See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details. | +| [`enableNodePublicIP`](#parameter-enablenodepublicip) | bool | Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools). | +| [`enableUltraSSD`](#parameter-enableultrassd) | bool | Whether to enable UltraSSD. | +| [`gpuInstanceProfile`](#parameter-gpuinstanceprofile) | string | GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU. | +| [`kubeletDiskType`](#parameter-kubeletdisktype) | string | Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage. | +| [`maxCount`](#parameter-maxcount) | int | The maximum number of nodes for auto-scaling. | +| [`maxPods`](#parameter-maxpods) | int | The maximum number of pods that can run on a node. | +| [`maxSurge`](#parameter-maxsurge) | string | This can either be set to an integer (e.g. "5") or a percentage (e.g. "50%"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade. | +| [`minCount`](#parameter-mincount) | int | The minimum number of nodes for auto-scaling. | +| [`mode`](#parameter-mode) | string | A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools. | +| [`nodeLabels`](#parameter-nodelabels) | object | The node labels to be persisted across all nodes in agent pool. | +| [`nodePublicIpPrefixId`](#parameter-nodepublicipprefixid) | string | ResourceId of the node PublicIPPrefix. | +| [`nodeTaints`](#parameter-nodetaints) | array | The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. | +| [`orchestratorVersion`](#parameter-orchestratorversion) | string | As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool). | +| [`osDiskSizeGB`](#parameter-osdisksizegb) | int | OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified. | +| [`osDiskType`](#parameter-osdisktype) | string | The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os). | +| [`osSku`](#parameter-ossku) | string | Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows. | +| [`osType`](#parameter-ostype) | string | The operating system type. The default is Linux. | +| [`podSubnetId`](#parameter-podsubnetid) | string | Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. | +| [`proximityPlacementGroupResourceId`](#parameter-proximityplacementgroupresourceid) | string | The ID for the Proximity Placement Group. | +| [`scaleDownMode`](#parameter-scaledownmode) | string | Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing). | +| [`scaleSetEvictionPolicy`](#parameter-scalesetevictionpolicy) | string | The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs. | +| [`scaleSetPriority`](#parameter-scalesetpriority) | string | The Virtual Machine Scale Set priority. | +| [`sourceResourceId`](#parameter-sourceresourceid) | string | This is the ARM ID of the source object to be used to create the target object. | +| [`spotMaxPrice`](#parameter-spotmaxprice) | int | Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing). | +| [`tags`](#parameter-tags) | object | Tags of the resource. | +| [`type`](#parameter-type) | string | The type of Agent Pool. | +| [`vmSize`](#parameter-vmsize) | string | VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions. | +| [`vnetSubnetId`](#parameter-vnetsubnetid) | string | Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. | +| [`workloadRuntime`](#parameter-workloadruntime) | string | Determines the type of workload a node can run. | + +### Parameter: `name` + +Name of the agent pool. + +- Required: Yes +- Type: string + +### Parameter: `managedClusterName` + +The name of the parent managed cluster. Required if the template is used in a standalone deployment. + +- Required: Yes +- Type: string + +### Parameter: `availabilityZones` + +The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets". + +- Required: No +- Type: array + +### Parameter: `count` + +Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1. + +- Required: No +- Type: int +- Default: `1` + +### Parameter: `enableAutoScaling` + +Whether to enable auto-scaler. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableEncryptionAtHost` + +This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableFIPS` + +See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableNodePublicIP` + +Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools). + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableUltraSSD` + +Whether to enable UltraSSD. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `gpuInstanceProfile` + +GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'MIG1g' + 'MIG2g' + 'MIG3g' + 'MIG4g' + 'MIG7g' + ] + ``` + +### Parameter: `kubeletDiskType` + +Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage. + +- Required: No +- Type: string + +### Parameter: `maxCount` + +The maximum number of nodes for auto-scaling. + +- Required: No +- Type: int + +### Parameter: `maxPods` + +The maximum number of pods that can run on a node. + +- Required: No +- Type: int + +### Parameter: `maxSurge` + +This can either be set to an integer (e.g. "5") or a percentage (e.g. "50%"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade. + +- Required: No +- Type: string + +### Parameter: `minCount` + +The minimum number of nodes for auto-scaling. + +- Required: No +- Type: int + +### Parameter: `mode` + +A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools. + +- Required: No +- Type: string + +### Parameter: `nodeLabels` + +The node labels to be persisted across all nodes in agent pool. + +- Required: No +- Type: object + +### Parameter: `nodePublicIpPrefixId` + +ResourceId of the node PublicIPPrefix. + +- Required: No +- Type: string + +### Parameter: `nodeTaints` + +The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. + +- Required: No +- Type: array + +### Parameter: `orchestratorVersion` + +As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool). + +- Required: No +- Type: string + +### Parameter: `osDiskSizeGB` + +OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified. + +- Required: No +- Type: int + +### Parameter: `osDiskType` + +The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os). + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Ephemeral' + 'Managed' + ] + ``` + +### Parameter: `osSku` + +Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'AzureLinux' + 'CBLMariner' + 'Ubuntu' + 'Windows2019' + 'Windows2022' + ] + ``` + +### Parameter: `osType` + +The operating system type. The default is Linux. + +- Required: No +- Type: string +- Default: `'Linux'` +- Allowed: + ```Bicep + [ + 'Linux' + 'Windows' + ] + ``` + +### Parameter: `podSubnetId` + +Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. + +- Required: No +- Type: string + +### Parameter: `proximityPlacementGroupResourceId` + +The ID for the Proximity Placement Group. + +- Required: No +- Type: string + +### Parameter: `scaleDownMode` + +Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing). + +- Required: No +- Type: string +- Default: `'Delete'` +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `scaleSetEvictionPolicy` + +The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs. + +- Required: No +- Type: string +- Default: `'Delete'` +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `scaleSetPriority` + +The Virtual Machine Scale Set priority. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Regular' + 'Spot' + ] + ``` + +### Parameter: `sourceResourceId` + +This is the ARM ID of the source object to be used to create the target object. + +- Required: No +- Type: string + +### Parameter: `spotMaxPrice` + +Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing). + +- Required: No +- Type: int + +### Parameter: `tags` + +Tags of the resource. + +- Required: No +- Type: object + +### Parameter: `type` + +The type of Agent Pool. + +- Required: No +- Type: string + +### Parameter: `vmSize` + +VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions. + +- Required: No +- Type: string +- Default: `'Standard_D2s_v3'` + +### Parameter: `vnetSubnetId` + +Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. + +- Required: No +- Type: string + +### Parameter: `workloadRuntime` + +Determines the type of workload a node can run. + +- Required: No +- Type: string + +## Outputs + +| Output | Type | Description | +| :-- | :-- | :-- | +| `name` | string | The name of the agent pool. | +| `resourceGroupName` | string | The resource group the agent pool was deployed into. | +| `resourceId` | string | The resource ID of the agent pool. | diff --git a/bicep/modules/managed-cluster/agent-pool/main.bicep b/bicep/modules/managed-cluster/agent-pool/main.bicep new file mode 100644 index 00000000..920a9000 --- /dev/null +++ b/bicep/modules/managed-cluster/agent-pool/main.bicep @@ -0,0 +1,217 @@ +metadata name = 'Azure Kubernetes Service (AKS) Managed Cluster Agent Pools' +metadata description = 'This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.' +metadata owner = 'Azure/module-maintainers' + +@description('Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment.') +param managedClusterName string + +@description('Required. Name of the agent pool.') +param name string + +@description('Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets".') +param availabilityZones array? + +@description('Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1.') +@minValue(0) +@maxValue(1000) +param count int = 1 + +@description('Optional. This is the ARM ID of the source object to be used to create the target object.') +param sourceResourceId string? + +@description('Optional. Whether to enable auto-scaler.') +param enableAutoScaling bool = false + +@description('Optional. This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled.') +param enableEncryptionAtHost bool = false + +@description('Optional. See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details.') +param enableFIPS bool = false + +@description('Optional. Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools).') +param enableNodePublicIP bool = false + +@description('Optional. Whether to enable UltraSSD.') +param enableUltraSSD bool = false + +@description('Optional. GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU.') +@allowed([ + 'MIG1g' + 'MIG2g' + 'MIG3g' + 'MIG4g' + 'MIG7g' +]) +param gpuInstanceProfile string? + +@description('Optional. Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage.') +param kubeletDiskType string? + +@description('Optional. The maximum number of nodes for auto-scaling.') +param maxCount int? + +@description('Optional. The maximum number of pods that can run on a node.') +param maxPods int? + +@description('Optional. The minimum number of nodes for auto-scaling.') +param minCount int? + +@description('Optional. A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools.') +param mode string? + +@description('Optional. The node labels to be persisted across all nodes in agent pool.') +param nodeLabels object? + +@description('Optional. ResourceId of the node PublicIPPrefix.') +param nodePublicIpPrefixId string? + +@description('Optional. The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule.') +param nodeTaints array? + +@description('Optional. As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool).') +param orchestratorVersion string? + +@description('Optional. OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified.') +param osDiskSizeGB int? + +@description('Optional. The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os).') +@allowed([ + 'Ephemeral' + 'Managed' +]) +param osDiskType string? + +@description('Optional. Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows.') +@allowed([ + 'AzureLinux' + 'CBLMariner' + 'Ubuntu' + 'Windows2019' + 'Windows2022' +]) +param osSku string? + +@description('Optional. The operating system type. The default is Linux.') +@allowed([ + 'Linux' + 'Windows' +]) +param osType string = 'Linux' + +@description('Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}.') +param podSubnetId string? + +@description('Optional. The ID for the Proximity Placement Group.') +param proximityPlacementGroupResourceId string? + +@description('Optional. Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing).') +@allowed([ + 'Deallocate' + 'Delete' +]) +param scaleDownMode string = 'Delete' + +@description('Optional. The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs.') +@allowed([ + 'Deallocate' + 'Delete' +]) +param scaleSetEvictionPolicy string = 'Delete' + +@description('Optional. The Virtual Machine Scale Set priority.') +@allowed([ + 'Regular' + 'Spot' +]) +param scaleSetPriority string? + +@description('Optional. Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing).') +param spotMaxPrice int? + +@description('Optional. Tags of the resource.') +param tags object? + +@description('Optional. The type of Agent Pool.') +param type string? + +@description('Optional. This can either be set to an integer (e.g. "5") or a percentage (e.g. "50%"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade.') +param maxSurge string? + +@description('Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions.') +param vmSize string = 'Standard_D2s_v3' + +@description('Optional. Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}.') +param vnetSubnetId string? + +@description('Optional. Determines the type of workload a node can run.') +param workloadRuntime string? + +@description('Optional. Whether to enable SSH access to the nodes.') +@allowed([ + 'LocalUser' + 'Disabled' +]) +param sshAccess string = 'Disabled' + +resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-preview' existing = { + name: managedClusterName +} + +resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = { + name: name + parent: managedCluster + properties: { + availabilityZones: availabilityZones + count: count + creationData: !empty(sourceResourceId) + ? { + sourceResourceId: sourceResourceId + } + : null + enableAutoScaling: enableAutoScaling + enableEncryptionAtHost: enableEncryptionAtHost + enableFIPS: enableFIPS + enableNodePublicIP: enableNodePublicIP + enableUltraSSD: enableUltraSSD + gpuInstanceProfile: gpuInstanceProfile + kubeletDiskType: kubeletDiskType + maxCount: maxCount + maxPods: maxPods + minCount: minCount + mode: mode + nodeLabels: nodeLabels + nodePublicIPPrefixID: nodePublicIpPrefixId + nodeTaints: nodeTaints + orchestratorVersion: orchestratorVersion + osDiskSizeGB: osDiskSizeGB + osDiskType: osDiskType + osSKU: osSku + osType: osType + podSubnetID: podSubnetId + proximityPlacementGroupID: proximityPlacementGroupResourceId + scaleDownMode: scaleDownMode + scaleSetEvictionPolicy: scaleSetEvictionPolicy + scaleSetPriority: scaleSetPriority + securityProfile: { + sshAccess: sshAccess + } + spotMaxPrice: spotMaxPrice + tags: tags + type: type + upgradeSettings: { + maxSurge: maxSurge + } + vmSize: vmSize + vnetSubnetID: vnetSubnetId + workloadRuntime: workloadRuntime + } +} + +@description('The name of the agent pool.') +output name string = agentPool.name + +@description('The resource ID of the agent pool.') +output resourceId string = agentPool.id + +@description('The resource group the agent pool was deployed into.') +output resourceGroupName string = resourceGroup().name diff --git a/bicep/modules/managed-cluster/agent-pool/main.json b/bicep/modules/managed-cluster/agent-pool/main.json new file mode 100644 index 00000000..d8141c2a --- /dev/null +++ b/bicep/modules/managed-cluster/agent-pool/main.json @@ -0,0 +1,380 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.29.47.4906", + "templateHash": "2004205618690542488" + }, + "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", + "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "managedClusterName": { + "type": "string", + "metadata": { + "description": "Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment." + } + }, + "name": { + "type": "string", + "metadata": { + "description": "Required. Name of the agent pool." + } + }, + "availabilityZones": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is \"VirtualMachineScaleSets\"." + } + }, + "count": { + "type": "int", + "defaultValue": 1, + "minValue": 0, + "maxValue": 1000, + "metadata": { + "description": "Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1." + } + }, + "sourceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. This is the ARM ID of the source object to be used to create the target object." + } + }, + "enableAutoScaling": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable auto-scaler." + } + }, + "enableEncryptionAtHost": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled." + } + }, + "enableFIPS": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details." + } + }, + "enableNodePublicIP": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools)." + } + }, + "enableUltraSSD": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable UltraSSD." + } + }, + "gpuInstanceProfile": { + "type": "string", + "nullable": true, + "allowedValues": [ + "MIG1g", + "MIG2g", + "MIG3g", + "MIG4g", + "MIG7g" + ], + "metadata": { + "description": "Optional. GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU." + } + }, + "kubeletDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage." + } + }, + "maxCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of nodes for auto-scaling." + } + }, + "maxPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of pods that can run on a node." + } + }, + "minCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of nodes for auto-scaling." + } + }, + "mode": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. A cluster must have at least one \"System\" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools." + } + }, + "nodeLabels": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The node labels to be persisted across all nodes in agent pool." + } + }, + "nodePublicIpPrefixId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. ResourceId of the node PublicIPPrefix." + } + }, + "nodeTaints": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule." + } + }, + "orchestratorVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool)." + } + }, + "osDiskSizeGB": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified." + } + }, + "osDiskType": { + "type": "string", + "nullable": true, + "allowedValues": [ + "Ephemeral", + "Managed" + ], + "metadata": { + "description": "Optional. The default is \"Ephemeral\" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to \"Managed\". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os)." + } + }, + "osSku": { + "type": "string", + "nullable": true, + "allowedValues": [ + "AzureLinux", + "CBLMariner", + "Ubuntu", + "Windows2019", + "Windows2022" + ], + "metadata": { + "description": "Optional. Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows." + } + }, + "osType": { + "type": "string", + "defaultValue": "Linux", + "allowedValues": [ + "Linux", + "Windows" + ], + "metadata": { + "description": "Optional. The operating system type. The default is Linux." + } + }, + "podSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + } + }, + "proximityPlacementGroupResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The ID for the Proximity Placement Group." + } + }, + "scaleDownMode": { + "type": "string", + "defaultValue": "Delete", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "metadata": { + "description": "Optional. Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing)." + } + }, + "scaleSetEvictionPolicy": { + "type": "string", + "defaultValue": "Delete", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "metadata": { + "description": "Optional. The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs." + } + }, + "scaleSetPriority": { + "type": "string", + "nullable": true, + "allowedValues": [ + "Regular", + "Spot" + ], + "metadata": { + "description": "Optional. The Virtual Machine Scale Set priority." + } + }, + "spotMaxPrice": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing)." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Tags of the resource." + } + }, + "type": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The type of Agent Pool." + } + }, + "maxSurge": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. This can either be set to an integer (e.g. \"5\") or a percentage (e.g. \"50%\"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade." + } + }, + "vmSize": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions." + } + }, + "vnetSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + } + }, + "workloadRuntime": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Determines the type of workload a node can run." + } + } + }, + "resources": { + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2024-03-02-preview", + "name": "[parameters('managedClusterName')]" + }, + "agentPool": { + "type": "Microsoft.ContainerService/managedClusters/agentPools", + "apiVersion": "2023-07-02-preview", + "name": "[format('{0}/{1}', parameters('managedClusterName'), parameters('name'))]", + "properties": { + "availabilityZones": "[parameters('availabilityZones')]", + "count": "[parameters('count')]", + "creationData": "[if(not(empty(parameters('sourceResourceId'))), createObject('sourceResourceId', parameters('sourceResourceId')), null())]", + "enableAutoScaling": "[parameters('enableAutoScaling')]", + "enableEncryptionAtHost": "[parameters('enableEncryptionAtHost')]", + "enableFIPS": "[parameters('enableFIPS')]", + "enableNodePublicIP": "[parameters('enableNodePublicIP')]", + "enableUltraSSD": "[parameters('enableUltraSSD')]", + "gpuInstanceProfile": "[parameters('gpuInstanceProfile')]", + "kubeletDiskType": "[parameters('kubeletDiskType')]", + "maxCount": "[parameters('maxCount')]", + "maxPods": "[parameters('maxPods')]", + "minCount": "[parameters('minCount')]", + "mode": "[parameters('mode')]", + "nodeLabels": "[parameters('nodeLabels')]", + "nodePublicIPPrefixID": "[parameters('nodePublicIpPrefixId')]", + "nodeTaints": "[parameters('nodeTaints')]", + "orchestratorVersion": "[parameters('orchestratorVersion')]", + "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "osDiskType": "[parameters('osDiskType')]", + "osSKU": "[parameters('osSku')]", + "osType": "[parameters('osType')]", + "podSubnetID": "[parameters('podSubnetId')]", + "proximityPlacementGroupID": "[parameters('proximityPlacementGroupResourceId')]", + "scaleDownMode": "[parameters('scaleDownMode')]", + "scaleSetEvictionPolicy": "[parameters('scaleSetEvictionPolicy')]", + "scaleSetPriority": "[parameters('scaleSetPriority')]", + "spotMaxPrice": "[parameters('spotMaxPrice')]", + "tags": "[parameters('tags')]", + "type": "[parameters('type')]", + "upgradeSettings": { + "maxSurge": "[parameters('maxSurge')]" + }, + "vmSize": "[parameters('vmSize')]", + "vnetSubnetID": "[parameters('vnetSubnetId')]", + "workloadRuntime": "[parameters('workloadRuntime')]" + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the agent pool." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the agent pool." + }, + "value": "[resourceId('Microsoft.ContainerService/managedClusters/agentPools', parameters('managedClusterName'), parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The resource group the agent pool was deployed into." + }, + "value": "[resourceGroup().name]" + } + } +} \ No newline at end of file diff --git a/bicep/modules/managed-cluster/aks_appconfig_extension.bicep b/bicep/modules/managed-cluster/aks_appconfig_extension.bicep new file mode 100644 index 00000000..59406d80 --- /dev/null +++ b/bicep/modules/managed-cluster/aks_appconfig_extension.bicep @@ -0,0 +1,18 @@ +@description('The name of the Managed Cluster resource.') +param clusterName string + +resource existingManagedCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-preview' existing = { + name: clusterName +} + +resource appConfigExtension 'Microsoft.KubernetesConfiguration/extensions@2023-05-01' = { + name: 'appconfigurationkubernetesprovider' + scope: existingManagedCluster + properties: { + autoUpgradeMinorVersion: true + configurationSettings: { + 'global.clusterType': 'managedclusters' + } + extensionType: 'microsoft.appconfiguration' + } +} diff --git a/bicep/modules/managed-cluster/aks_policy.bicep b/bicep/modules/managed-cluster/aks_policy.bicep new file mode 100644 index 00000000..f0da2abc --- /dev/null +++ b/bicep/modules/managed-cluster/aks_policy.bicep @@ -0,0 +1,42 @@ +@description('The name of the Azure Kubernetes Service Cluster') +param clusterName string = '' + +resource managedCluster 'Microsoft.ContainerService/managedClusters@2023-05-02-preview' existing = if (clusterName != '') { + name: clusterName +} + +var policyDefinitionId = '/providers/Microsoft.Authorization/policySetDefinitions/c047ea8e-9c78-49b2-958b-37e56d291a44' +resource policyAssignment 'Microsoft.Authorization/policyAssignments@2024-04-01' = { + name: 'aksDeploymentSafeguardsAssignment' + scope: managedCluster + properties: { + displayName: 'AKS Deployment Safeguards' + #disable-next-line use-resource-id-functions + policyDefinitionId: policyDefinitionId + enforcementMode: 'DoNotEnforce' + parameters: { + effect: { value: 'Audit' } + allowedUsers: { + value: [] // Specify allowed users or leave empty array + } + allowedGroups: { + value: [] // Specify allowed groups or leave empty array + } + cpuLimit: { + value: '4' // Specify CPU limit, e.g., '1' for 1 core + } + memoryLimit: { + value: '4Gi' // Specify memory limit, e.g., '1Gi' for 1 Gibibyte + } + labels: { + value: [] // Specify required labels or leave empty object + } + allowedContainerImagesRegex: { + value: '.*' // Specify regex for allowed container images, e.g., '.*' to allow all + } + reservedTaints: { + value: [] // Specify reserved taints or leave empty array + } + } + } +} diff --git a/bicep/modules/managed-cluster/main.bicep b/bicep/modules/managed-cluster/main.bicep new file mode 100644 index 00000000..e5d0f8f8 --- /dev/null +++ b/bicep/modules/managed-cluster/main.bicep @@ -0,0 +1,1274 @@ +metadata name = 'Azure Kubernetes Service (AKS) Managed Clusters' +metadata description = 'This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.' + +@description('Required. Specifies the name of the AKS cluster.') +param name string + +@description('Optional. Specifies the location of AKS cluster. It picks up Resource Group\'s location by default.') +param location string = resourceGroup().location + +@description('Optional. Specifies the DNS prefix specified when creating the managed cluster.') +param dnsPrefix string = name + +@description('Optional. The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both.') +param managedIdentities managedIdentitiesType + +@description('Optional. Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin.') +@allowed([ + 'azure' + 'cilium' +]) +param networkDataplane string? + +@description('Optional. Specifies the network plugin used for building Kubernetes network.') +@allowed([ + 'azure' + 'kubenet' +]) +param networkPlugin string? + +@description('Optional. Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin.') +@allowed([ + 'overlay' +]) +param networkPluginMode string? + +@description('Optional. Specifies the network policy used for building Kubernetes network. - calico or azure.') +@allowed([ + 'azure' + 'calico' +]) +param networkPolicy string? + +@description('Optional. Specifies the CIDR notation IP range from which to assign pod IPs when kubenet is used.') +param podCidr string? + +@description('Optional. A CIDR notation IP range from which to assign service cluster IPs. It must not overlap with any Subnet IP ranges.') +param serviceCidr string? + +@description('Optional. Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr.') +param dnsServiceIP string? + +@description('Optional. Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools.') +@allowed([ + 'basic' + 'standard' +]) +param loadBalancerSku string = 'standard' + +@description('Optional. Outbound IP Count for the Load balancer.') +param managedOutboundIPCount int = 0 + +@description('Optional. Outbound IP Resource IDs.') +param outboundIPResourceIds array = [] + +@description('Optional. The type of the managed inbound Load Balancer BackendPool.') +@allowed([ + 'NodeIP' + 'NodeIPConfiguration' +]) +param backendPoolType string = 'NodeIPConfiguration' + +@description('Optional. Specifies outbound (egress) routing method.') +@allowed([ + 'loadBalancer' + 'userDefinedRouting' + 'managedNATGateway' + 'userAssignedNATGateway' +]) +param outboundType string = 'loadBalancer' + +@description('Optional. Tier of a managed cluster SKU.') +@allowed([ + 'Free' + 'Premium' + 'Standard' +]) +param skuTier string = 'Standard' + +@description('Optional. AKS Name.') +@allowed([ + 'Base' + 'Automatic' +]) +param skuName string = 'Base' + +@description('Optional. Version of Kubernetes specified when creating the managed cluster.') +param kubernetesVersion string? + +@description('Optional. Specifies the administrator username of Linux virtual machines.') +param adminUsername string = 'azureuser' + +@description('Optional. Specifies the SSH RSA public key string for the Linux nodes.') +param sshPublicKey string? + +@description('Conditional. Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster.') +param aksServicePrincipalProfile object? + +@description('Optional. The client AAD application ID.') +param aadProfileClientAppID string? + +@description('Optional. The server AAD application ID.') +param aadProfileServerAppID string? + +@description('Optional. The server AAD application secret.') +#disable-next-line secure-secrets-in-params // Not a secret +param aadProfileServerAppSecret string? + +@description('Optional. Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication.') +param aadProfileTenantId string = subscription().tenantId + +@description('Optional. Specifies the AAD group object IDs that will have admin role of the cluster.') +param aadProfileAdminGroupObjectIDs array? + +@description('Optional. Specifies whether to enable managed AAD integration.') +param aadProfileManaged bool = true + +@description('Optional. Whether to enable Kubernetes Role-Based Access Control.') +param enableRBAC bool = true + +@description('Optional. Specifies whether to enable Azure RBAC for Kubernetes authorization.') +param aadProfileEnableAzureRBAC bool = enableRBAC + +@description('Optional. If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled.') +param disableLocalAccounts bool = false + +@description('Optional. Name of the resource group containing agent pool nodes.') +param nodeResourceGroup string = '${resourceGroup().name}_aks_${name}_nodes' + +@description('Optional. If set to true, the node resource group will be locked down to prevent accidental deletion.') +param nodeResourceGroupLockDown bool = false + +@description('Optional. IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer.') +param authorizedIPRanges array? + +@description('Optional. Whether to disable run command for the cluster or not.') +param disableRunCommand bool = false + +@description('Optional. Allow or deny public network access for AKS.') +@allowed([ + 'Enabled' + 'Disabled' + 'SecuredByPerimeter' +]) +param publicNetworkAccess string = 'Disabled' + +@description('Optional. Specifies whether to create the cluster as a private cluster or not.') +param enablePrivateCluster bool = false + +@description('Optional. Whether to create additional public FQDN for private cluster or not.') +param enablePrivateClusterPublicFQDN bool = false + +@description('Optional. Private DNS Zone configuration. Set to \'system\' and AKS will create a private DNS zone in the node resource group. Set to \'\' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone.') +param privateDNSZone string? + +@description('Required. Properties of the primary agent pool.') +param primaryAgentPoolProfile array + +@description('Optional. Define one or more secondary/additional agent pools.') +param agentPools agentPoolType + +@description('Optional. Whether or not to use AKS Automatic mode.') +param maintenanceConfiguration maintenanceConfigurationType + +@description('Optional. Specifies whether the cost analysis add-on is enabled or not. If Enabled `enableStorageProfileDiskCSIDriver` is set to true as it is needed.') +param costAnalysisEnabled bool = false + +@description('Optional. Specifies whether the httpApplicationRouting add-on is enabled or not.') +param httpApplicationRoutingEnabled bool = false + +@description('Optional. Specifies whether the webApplicationRoutingEnabled add-on is enabled or not.') +param webApplicationRoutingEnabled bool = false + +@description('Optional. Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`.') +param dnsZoneResourceId string? + +@description('Optional. Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided.') +param enableDnsZoneContributorRoleAssignment bool = true + +@description('Optional. Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not.') +param ingressApplicationGatewayEnabled bool = false + +@description('Conditional. Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`.') +param appGatewayResourceId string? + +@description('Optional. Specifies whether the aciConnectorLinux add-on is enabled or not.') +param aciConnectorLinuxEnabled bool = false + +@description('Optional. Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled.') +param azurePolicyEnabled bool = true + +@description('Optional. Specifies whether the openServiceMesh add-on is enabled or not.') +param openServiceMeshEnabled bool = false + +@description('Optional. Specifies whether the istioServiceMesh add-on is enabled or not.') +param istioServiceMeshEnabled bool = false + +@description('Specifies whether the Istio Ingress Gateway is enabled or not.') +param istioIngressGatewayEnabled bool = false + +@description('Specifies the type of the Istio Ingress Gateway.') +@allowed([ + 'Internal' + 'External' +]) +param istioIngressGatewayType string = 'External' + +@description('Optional. Specifies the Istio revision to use.') +param istioRevision string = 'asm-1-23' + +@description('Optional. Specifies the azure policy version to use.') +param azurePolicyVersion string = 'v2' + +@description('Optional. Specifies whether the kubeDashboard add-on is enabled or not.') +param kubeDashboardEnabled bool = false + +@description('Optional. Specifies whether the KeyvaultSecretsProvider add-on is enabled or not.') +#disable-next-line secure-secrets-in-params // Not a secret +param enableKeyvaultSecretsProvider bool = false + +@description('Optional. Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation.') +#disable-next-line secure-secrets-in-params // Not a secret +param enableSecretRotation bool = false + +@description('Optional. Specifies the scan interval of the auto-scaler of the AKS cluster.') +param autoScalerProfileScanInterval string = '10s' + +@description('Optional. Specifies the scale down delay after add of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownDelayAfterAdd string = '10m' + +@description('Optional. Specifies the scale down delay after delete of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownDelayAfterDelete string = '20s' + +@description('Optional. Specifies scale down delay after failure of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownDelayAfterFailure string = '3m' + +@description('Optional. Specifies the scale down unneeded time of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownUnneededTime string = '10m' + +@description('Optional. Specifies the scale down unready time of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownUnreadyTime string = '20m' + +@description('Optional. Specifies the utilization threshold of the auto-scaler of the AKS cluster.') +param autoScalerProfileUtilizationThreshold string = '0.5' + +@description('Optional. Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster.') +param autoScalerProfileMaxGracefulTerminationSec string = '600' + +@description('Optional. Specifies the balance of similar node groups for the auto-scaler of the AKS cluster.') +param autoScalerProfileBalanceSimilarNodeGroups bool = false + +@allowed([ + 'least-waste' + 'most-pods' + 'priority' + 'random' +]) +@description('Optional. Specifies the expand strategy for the auto-scaler of the AKS cluster.') +param autoScalerProfileExpander string = 'random' + +@description('Optional. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster.') +param autoScalerProfileMaxEmptyBulkDelete string = '10' + +@description('Optional. Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported.') +param autoScalerProfileMaxNodeProvisionTime string = '15m' + +@description('Optional. Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0.') +param autoScalerProfileMaxTotalUnreadyPercentage string = '45' + +@description('Optional. For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc).') +param autoScalerProfileNewPodScaleUpDelay string = '0s' + +@description('Optional. Specifies the OK total unready count for the auto-scaler of the AKS cluster.') +param autoScalerProfileOkTotalUnreadyCount string = '3' + +@description('Optional. Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster.') +param autoScalerProfileSkipNodesWithLocalStorage bool = true + +@description('Optional. Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster.') +param autoScalerProfileSkipNodesWithSystemPods bool = true + +@allowed([ + 'node-image' + 'none' + 'patch' + 'rapid' + 'stable' +]) +@description('Optional. Auto-upgrade channel on the AKS cluster.') +param autoUpgradeProfileUpgradeChannel string = 'stable' + +@description('Optional. Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing.') +param podIdentityProfileAllowNetworkPluginKubenet bool = false + +@description('Optional. Whether the pod identity addon is enabled.') +param podIdentityProfileEnable bool = false + +@description('Optional. The pod identities to use in the cluster.') +param podIdentityProfileUserAssignedIdentities array? + +@description('Optional. The pod identity exceptions to allow.') +param podIdentityProfileUserAssignedIdentityExceptions array? + +@description('Optional. Whether the The OIDC issuer profile of the Managed Cluster is enabled.') +param enableOidcIssuerProfile bool = false + +@description('Optional. Whether to enable Workload Identity. Requires OIDC issuer profile to be enabled.') +param enableWorkloadIdentity bool = false + +@description('Optional. Whether to enable Azure Defender.') +param enableAzureDefender bool = false + +@description('Optional. Whether to enable Image Cleaner for Kubernetes.') +param enableImageCleaner bool = false + +@description('Optional. The interval in hours Image Cleaner will run. The maximum value is three months.') +@minValue(24) +param imageCleanerIntervalHours int = 24 + +@description('Optional. Whether to enable Kubernetes pod security policy. Requires enabling the pod security policy feature flag on the subscription.') +param enablePodSecurityPolicy bool = false + +@description('Optional. Whether the AzureBlob CSI Driver for the storage profile is enabled.') +param enableStorageProfileBlobCSIDriver bool = false + +@description('Optional. Whether the AzureDisk CSI Driver for the storage profile is enabled.') +param enableStorageProfileDiskCSIDriver bool = false + +@description('Optional. Whether the AzureFile CSI Driver for the storage profile is enabled.') +param enableStorageProfileFileCSIDriver bool = false + +@description('Optional. Whether the snapshot controller for the storage profile is enabled.') +param enableStorageProfileSnapshotController bool = false + +@allowed([ + 'AKSLongTermSupport' + 'KubernetesOfficial' +]) +@description('Optional. The support plan for the Managed Cluster.') +param supportPlan string = 'KubernetesOfficial' + +@description('Optional. The diagnostic settings of the service.') +param diagnosticSettings diagnosticSettingType + +@description('Optional. Specifies whether the OMS agent is enabled.') +param omsAgentEnabled bool = true + +@description('Optional. Resource ID of the monitoring log analytics workspace.') +param monitoringWorkspaceId string? + +@description('Optional. Enable/Disable usage telemetry for module.') +param enableTelemetry bool = true + +@description('Optional. Array of role assignments to create.') +param roleAssignments roleAssignmentType + +@description('Optional. The lock settings of the service.') +param lock lockType + +@description('Optional. Tags of the resource.') +param tags object? + +@description('Optional. The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided.') +param diskEncryptionSetResourceId string? + +@description('Optional. Settings and configurations for the flux extension.') +param fluxExtension extensionType + +@description('Optional. Configurations for provisioning the cluster with HTTP proxy servers.') +param httpProxyConfig object? + +@description('Optional. Identities associated with the cluster.') +param identityProfile object? + +@description('Optional. Enables Kubernetes Event-driven Autoscaling (KEDA).') +param kedaAddon bool = false + +@description('Optional. Whether to enable VPA add-on in cluster. Default value is false.') +param vpaAddon bool = false + +@description('Optional. The customer managed key definition.') +param customerManagedKey customerManagedKeyType + +@description('Optional. Whether the metric state of the kubenetes cluster is enabled.') +param enableAzureMonitorProfileMetrics bool = false + +@description('Optional. Indicates if Azure Monitor Container Insights Logs Addon is enabled.') +param enableContainerInsights bool = false + +@description('Optional. Indicates whether custom metrics collection has to be disabled or not. If not specified the default is false. No custom metrics will be emitted if this field is false but the container insights enabled field is false.') +param disableCustomMetrics bool = false + +@description('Optional. Indicates whether prometheus metrics scraping is disabled or not. If not specified the default is false. No prometheus metrics will be emitted if this field is false but the container insights enabled field is false.') +param disablePrometheusMetricsScraping bool = false + +@description('Optional. The syslog host port. If not specified, the default port is 28330.') +param syslogPort int = 28330 + +@description('Optional. A comma-separated list of kubernetes cluster metrics labels.') +param metricLabelsAllowlist string = '' + +@description('Optional. A comma-separated list of Kubernetes cluster metrics annotations.') +param metricAnnotationsAllowList string = '' + +@description('Optional. Whether to enable node auto-provisioning.') +param enableNodeAutoProvisioning bool = false + +// =========== // +// Variables // +// =========== // + +var formattedUserAssignedIdentities = reduce( + map((managedIdentities.?userAssignedResourcesIds ?? []), (id) => { '${id}': {} }), + {}, + (cur, next) => union(cur, next) +) // Converts the flat array to an object like { '${id1}': {}, '${id2}': {} } + +var identity = !empty(managedIdentities) + ? { + type: (managedIdentities.?systemAssigned ?? false) + ? 'SystemAssigned' + : (!empty(managedIdentities.?userAssignedResourcesIds ?? {}) ? 'UserAssigned' : null) + userAssignedIdentities: !empty(formattedUserAssignedIdentities) ? formattedUserAssignedIdentities : null + } + : null + +var builtInRoleNames = { + 'Azure Kubernetes Fleet Manager Contributor Role': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '63bb64ad-9799-4770-b5c3-24ed299a07bf' + ) + 'Azure Kubernetes Fleet Manager RBAC Admin': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '434fb43a-c01c-447e-9f67-c3ad923cfaba' + ) + 'Azure Kubernetes Fleet Manager RBAC Cluster Admin': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '18ab4d3d-a1bf-4477-8ad9-8359bc988f69' + ) + 'Azure Kubernetes Fleet Manager RBAC Reader': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '30b27cfc-9c84-438e-b0ce-70e35255df80' + ) + 'Azure Kubernetes Fleet Manager RBAC Writer': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '5af6afb3-c06c-4fa4-8848-71a8aee05683' + ) + 'Azure Kubernetes Service Cluster Admin Role': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8' + ) + 'Azure Kubernetes Service Cluster Monitoring User': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '1afdec4b-e479-420e-99e7-f82237c7c5e6' + ) + 'Azure Kubernetes Service Cluster User Role': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '4abbcc35-e782-43d8-92c5-2d3f1bd2253f' + ) + 'Azure Kubernetes Service Contributor Role': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'ed7f3fbd-7b88-4dd4-9017-9adb7ce333f8' + ) + 'Azure Kubernetes Service RBAC Admin': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '3498e952-d568-435e-9b2c-8d77e338d7f7' + ) + 'Azure Kubernetes Service RBAC Cluster Admin': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b' + ) + 'Azure Kubernetes Service RBAC Reader': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '7f6c6a51-bcf8-42ba-9220-52d62157d7db' + ) + 'Azure Kubernetes Service RBAC Writer': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'a7ffa36f-339b-4b5c-8bdf-e2c188b2c0eb' + ) + Contributor: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') + 'Kubernetes Agentless Operator': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'd5a2ae44-610b-4500-93be-660a0c5f5ca6' + ) + Owner: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + Reader: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') + 'Role Based Access Control Administrator (Preview)': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'f58310d9-a9f6-439a-9e8d-f62e7b41a168' + ) + 'User Access Administrator': subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9' + ) +} + +var formattedRoleAssignments = [ + for (roleAssignment, index) in (roleAssignments ?? []): union(roleAssignment, { + roleDefinitionId: builtInRoleNames[?roleAssignment.roleDefinitionIdOrName] ?? (contains( + roleAssignment.roleDefinitionIdOrName, + '/providers/Microsoft.Authorization/roleDefinitions/' + ) + ? roleAssignment.roleDefinitionIdOrName + : subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleAssignment.roleDefinitionIdOrName)) + }) +] + +// ============ // +// Dependencies // +// ============ // + +#disable-next-line no-deployments-resources +resource avmTelemetry 'Microsoft.Resources/deployments@2024-03-01' = if (enableTelemetry) { + name: '46d3xbcp.res.containerservice-managedcluster.${replace('-..--..-', '.', '-')}.${substring(uniqueString(deployment().name, location), 0, 4)}' + properties: { + mode: 'Incremental' + template: { + '$schema': 'https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#' + contentVersion: '1.0.0.0' + resources: [] + outputs: { + telemetry: { + type: 'String' + value: 'For more information, see https://aka.ms/avm/TelemetryInfo' + } + } + } + } +} + +resource cMKKeyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = if (!empty(customerManagedKey.?keyVaultResourceId)) { + name: last(split((customerManagedKey.?keyVaultResourceId ?? 'dummyVault'), '/')) + scope: resourceGroup( + split((customerManagedKey.?keyVaultResourceId ?? '//'), '/')[2], + split((customerManagedKey.?keyVaultResourceId ?? '////'), '/')[4] + ) + + resource cMKKey 'keys@2023-02-01' existing = if (!empty(customerManagedKey.?keyVaultResourceId) && !empty(customerManagedKey.?keyName)) { + name: customerManagedKey.?keyName ?? 'dummyKey' + } +} + +// ============== // +// Main Resources // +// ============== // + +resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-preview' = { + name: name + location: location + tags: tags + identity: identity + sku: { + name: skuName + tier: skuTier + } + properties: { + httpProxyConfig: httpProxyConfig + identityProfile: identityProfile + diskEncryptionSetID: diskEncryptionSetResourceId + kubernetesVersion: kubernetesVersion + dnsPrefix: dnsPrefix + agentPoolProfiles: primaryAgentPoolProfile + linuxProfile: !empty(sshPublicKey) + ? { + adminUsername: adminUsername + ssh: { + publicKeys: [ + { + keyData: sshPublicKey ?? '' + } + ] + } + } + : null + servicePrincipalProfile: aksServicePrincipalProfile + metricsProfile: { + costAnalysis: { + enabled: skuTier == 'free' ? false : costAnalysisEnabled + } + } + ingressProfile: { + webAppRouting: { + enabled: webApplicationRoutingEnabled + dnsZoneResourceIds: !empty(dnsZoneResourceId) + ? [ + any(dnsZoneResourceId) + ] + : null + } + } + addonProfiles: { + httpApplicationRouting: { + enabled: httpApplicationRoutingEnabled + } + ingressApplicationGateway: { + enabled: ingressApplicationGatewayEnabled && !empty(appGatewayResourceId) + #disable-next-line BCP321 // Value will not be used if null or empty + config: ingressApplicationGatewayEnabled && !empty(appGatewayResourceId) + ? { + applicationGatewayId: appGatewayResourceId + effectiveApplicationGatewayId: appGatewayResourceId + } + : null + } + omsagent: { + enabled: omsAgentEnabled && !empty(monitoringWorkspaceId) + #disable-next-line BCP321 // Value will not be used if null or empty + config: omsAgentEnabled && !empty(monitoringWorkspaceId) + ? { + logAnalyticsWorkspaceResourceID: monitoringWorkspaceId + } + : null + } + aciConnectorLinux: { + enabled: aciConnectorLinuxEnabled + } + azurepolicy: { + enabled: azurePolicyEnabled + config: azurePolicyEnabled + ? { + version: azurePolicyVersion + } + : null + } + openServiceMesh: { + enabled: openServiceMeshEnabled + config: openServiceMeshEnabled ? {} : null + } + kubeDashboard: { + enabled: kubeDashboardEnabled + } + azureKeyvaultSecretsProvider: { + enabled: enableKeyvaultSecretsProvider + config: enableKeyvaultSecretsProvider + ? { + enableSecretRotation: toLower(string(enableSecretRotation)) + } + : null + } + } + oidcIssuerProfile: enableOidcIssuerProfile + ? { + enabled: enableOidcIssuerProfile + } + : null + enableRBAC: enableRBAC + disableLocalAccounts: disableLocalAccounts + nodeResourceGroup: nodeResourceGroup + nodeResourceGroupProfile: { + restrictionLevel: nodeResourceGroupLockDown ? 'ReadOnly' : 'Unrestricted' + } + enablePodSecurityPolicy: enablePodSecurityPolicy + workloadAutoScalerProfile: { + keda: { + enabled: kedaAddon + } + verticalPodAutoscaler: { + enabled: vpaAddon + } + } + networkProfile: { + networkDataplane: networkDataplane + networkPlugin: networkPlugin + networkPluginMode: networkPluginMode + networkPolicy: networkPolicy + podCidr: podCidr + serviceCidr: serviceCidr + dnsServiceIP: dnsServiceIP + outboundType: outboundType + loadBalancerSku: loadBalancerSku + loadBalancerProfile: { + managedOutboundIPs: managedOutboundIPCount != 0 + ? { + count: managedOutboundIPCount + } + : null + outboundIPs: !empty(outboundIPResourceIds) + ? { + publicIPs: map(outboundIPResourceIds, id => { + id: id + }) + } + : null + effectiveOutboundIPs: [] + backendPoolType: backendPoolType + } + } + publicNetworkAccess: publicNetworkAccess + aadProfile: { + clientAppID: aadProfileClientAppID + serverAppID: aadProfileServerAppID + serverAppSecret: aadProfileServerAppSecret + managed: aadProfileManaged + enableAzureRBAC: aadProfileEnableAzureRBAC + adminGroupObjectIDs: aadProfileAdminGroupObjectIDs + tenantID: aadProfileTenantId + } + autoScalerProfile: { + 'balance-similar-node-groups': toLower(string(autoScalerProfileBalanceSimilarNodeGroups)) + expander: autoScalerProfileExpander + 'max-empty-bulk-delete': autoScalerProfileMaxEmptyBulkDelete + 'max-graceful-termination-sec': autoScalerProfileMaxGracefulTerminationSec + 'max-node-provision-time': autoScalerProfileMaxNodeProvisionTime + 'max-total-unready-percentage': autoScalerProfileMaxTotalUnreadyPercentage + 'new-pod-scale-up-delay': autoScalerProfileNewPodScaleUpDelay + 'ok-total-unready-count': autoScalerProfileOkTotalUnreadyCount + 'scale-down-delay-after-add': autoScalerProfileScaleDownDelayAfterAdd + 'scale-down-delay-after-delete': autoScalerProfileScaleDownDelayAfterDelete + 'scale-down-delay-after-failure': autoScalerProfileScaleDownDelayAfterFailure + 'scale-down-unneeded-time': autoScalerProfileScaleDownUnneededTime + 'scale-down-unready-time': autoScalerProfileScaleDownUnreadyTime + 'scale-down-utilization-threshold': autoScalerProfileUtilizationThreshold + 'scan-interval': autoScalerProfileScanInterval + 'skip-nodes-with-local-storage': toLower(string(autoScalerProfileSkipNodesWithLocalStorage)) + 'skip-nodes-with-system-pods': toLower(string(autoScalerProfileSkipNodesWithSystemPods)) + } + autoUpgradeProfile: { + upgradeChannel: autoUpgradeProfileUpgradeChannel + } + apiServerAccessProfile: { + authorizedIPRanges: authorizedIPRanges + disableRunCommand: disableRunCommand + enablePrivateCluster: enablePrivateCluster + enablePrivateClusterPublicFQDN: enablePrivateClusterPublicFQDN + privateDNSZone: privateDNSZone + } + azureMonitorProfile: { + containerInsights: enableContainerInsights + ? { + enabled: enableContainerInsights + logAnalyticsWorkspaceResourceId: !empty(monitoringWorkspaceId) ? monitoringWorkspaceId : null + disableCustomMetrics: disableCustomMetrics + disablePrometheusMetricsScraping: disablePrometheusMetricsScraping + syslogPort: syslogPort + } + : null + metrics: enableAzureMonitorProfileMetrics + ? { + enabled: enableAzureMonitorProfileMetrics + kubeStateMetrics: { + metricLabelsAllowlist: metricLabelsAllowlist + metricAnnotationsAllowList: metricAnnotationsAllowList + } + } + : null + } + podIdentityProfile: { + allowNetworkPluginKubenet: podIdentityProfileAllowNetworkPluginKubenet + enabled: podIdentityProfileEnable + userAssignedIdentities: podIdentityProfileUserAssignedIdentities + userAssignedIdentityExceptions: podIdentityProfileUserAssignedIdentityExceptions + } + securityProfile: { + defender: enableAzureDefender + ? { + securityMonitoring: { + enabled: enableAzureDefender + } + logAnalyticsWorkspaceResourceId: monitoringWorkspaceId + } + : null + workloadIdentity: enableWorkloadIdentity + ? { + enabled: enableWorkloadIdentity + } + : null + imageCleaner: enableImageCleaner + ? { + enabled: enableImageCleaner + intervalHours: imageCleanerIntervalHours + } + : null + } + serviceMeshProfile: istioServiceMeshEnabled ? { + istio: { + components: { + ingressGateways: istioIngressGatewayEnabled ? [ + { + enabled: true + mode: istioIngressGatewayType + } + ] : null + } + revisions: [ + istioRevision + ] + } + mode: 'Istio' + } : null + storageProfile: { + blobCSIDriver: { + enabled: enableStorageProfileBlobCSIDriver + } + diskCSIDriver: { + enabled: costAnalysisEnabled == true && skuTier != 'free' ? true : enableStorageProfileDiskCSIDriver + } + fileCSIDriver: { + enabled: enableStorageProfileFileCSIDriver + } + snapshotController: { + enabled: enableStorageProfileSnapshotController + } + } + supportPlan: supportPlan + nodeProvisioningProfile: enableNodeAutoProvisioning ? { + mode: 'Auto' + } : null + } +} + +module managedCluster_maintenanceConfigurations 'maintenance-configurations/main.bicep' = if (!empty(maintenanceConfiguration)) { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-MaintenanceConfigurations' + params: { + maintenanceWindow: maintenanceConfiguration!.maintenanceWindow + managedClusterName: managedCluster.name + } +} + +module managedCluster_agentPools 'agent-pool/main.bicep' = [ + for (agentPool, index) in (agentPools ?? []): { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-AgentPool-${index}' + params: { + managedClusterName: managedCluster.?name + name: agentPool.name + availabilityZones: agentPool.?availabilityZones + count: agentPool.?count + sourceResourceId: agentPool.?sourceResourceId + enableAutoScaling: agentPool.?enableAutoScaling + enableEncryptionAtHost: agentPool.?enableEncryptionAtHost + enableFIPS: agentPool.?enableFIPS + enableNodePublicIP: agentPool.?enableNodePublicIP + enableUltraSSD: agentPool.?enableUltraSSD + gpuInstanceProfile: agentPool.?gpuInstanceProfile + kubeletDiskType: agentPool.?kubeletDiskType + maxCount: agentPool.?maxCount + maxPods: agentPool.?maxPods + minCount: agentPool.?minCount + mode: agentPool.?mode + nodeLabels: agentPool.?nodeLabels + nodePublicIpPrefixId: agentPool.?nodePublicIpPrefixId + nodeTaints: agentPool.?nodeTaints + orchestratorVersion: agentPool.?orchestratorVersion ?? kubernetesVersion + osDiskSizeGB: agentPool.?osDiskSizeGB + osDiskType: agentPool.?osDiskType + osSku: agentPool.?osSku + osType: agentPool.?osType + podSubnetId: agentPool.?podSubnetId + proximityPlacementGroupResourceId: agentPool.?proximityPlacementGroupResourceId + scaleDownMode: agentPool.?scaleDownMode + scaleSetEvictionPolicy: agentPool.?scaleSetEvictionPolicy + scaleSetPriority: agentPool.?scaleSetPriority + spotMaxPrice: agentPool.?spotMaxPrice + sshAccess: agentPool.?sshAccess + tags: agentPool.?tags ?? tags + type: agentPool.?type + maxSurge: agentPool.?maxSurge + vmSize: agentPool.?vmSize + vnetSubnetId: agentPool.?vnetSubnetId + workloadRuntime: agentPool.?workloadRuntime + } + } +] + +module managedCluster_extension 'br/public:avm/res/kubernetes-configuration/extension:0.2.0' = if (!empty(fluxExtension)) { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' + params: { + clusterName: managedCluster.name + configurationProtectedSettings: fluxExtension.?configurationProtectedSettings + configurationSettings: fluxExtension.?configurationSettings + enableTelemetry: enableTelemetry + extensionType: 'microsoft.flux' + fluxConfigurations: fluxExtension.?configurations + location: location + name: 'flux' + releaseNamespace: fluxExtension.?releaseNamespace ?? 'flux-system' + releaseTrain: fluxExtension.?releaseTrain ?? 'Stable' + version: fluxExtension.?version + } +} + +resource managedCluster_lock 'Microsoft.Authorization/locks@2020-05-01' = if (!empty(lock ?? {}) && lock.?kind != 'None') { + name: lock.?name ?? 'lock-${name}' + properties: { + level: lock.?kind ?? '' + notes: lock.?kind == 'CanNotDelete' + ? 'Cannot delete resource or child resources.' + : 'Cannot delete or modify the resource or child resources.' + } + scope: managedCluster +} + +resource managedCluster_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = [ + for (diagnosticSetting, index) in (diagnosticSettings ?? []): { + name: diagnosticSetting.?name ?? '${name}-diagnosticSettings' + properties: { + storageAccountId: diagnosticSetting.?storageAccountResourceId + workspaceId: diagnosticSetting.?workspaceResourceId + eventHubAuthorizationRuleId: diagnosticSetting.?eventHubAuthorizationRuleResourceId + eventHubName: diagnosticSetting.?eventHubName + metrics: [ + for group in (diagnosticSetting.?metricCategories ?? [{ category: 'AllMetrics' }]): { + category: group.category + enabled: group.?enabled ?? true + timeGrain: null + } + ] + logs: [ + for group in (diagnosticSetting.?logCategoriesAndGroups ?? [{ categoryGroup: 'allLogs' }]): { + categoryGroup: group.?categoryGroup + category: group.?category + enabled: group.?enabled ?? true + } + ] + marketplacePartnerId: diagnosticSetting.?marketplacePartnerResourceId + logAnalyticsDestinationType: diagnosticSetting.?logAnalyticsDestinationType + } + scope: managedCluster + } +] + +resource managedCluster_roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ + for (roleAssignment, index) in (formattedRoleAssignments ?? []): { + name: roleAssignment.?name ?? guid(managedCluster.id, roleAssignment.principalId, roleAssignment.roleDefinitionId) + properties: { + roleDefinitionId: roleAssignment.roleDefinitionId + principalId: roleAssignment.principalId + description: roleAssignment.?description + principalType: roleAssignment.?principalType + condition: roleAssignment.?condition + conditionVersion: !empty(roleAssignment.?condition) ? (roleAssignment.?conditionVersion ?? '2.0') : null // Must only be set if condtion is set + delegatedManagedIdentityResourceId: roleAssignment.?delegatedManagedIdentityResourceId + } + scope: managedCluster + } +] + +resource dnsZone 'Microsoft.Network/dnsZones@2018-05-01' existing = if (enableDnsZoneContributorRoleAssignment == true && dnsZoneResourceId != null && webApplicationRoutingEnabled) { + name: last(split((!empty(dnsZoneResourceId) ? any(dnsZoneResourceId) : '/dummmyZone'), '/'))! +} + +resource dnsZone_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (enableDnsZoneContributorRoleAssignment == true && dnsZoneResourceId != null && webApplicationRoutingEnabled) { + name: guid( + dnsZone.id, + subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314'), + 'DNS Zone Contributor' + ) + properties: { + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'befefa01-2a29-4197-83a8-272ff33ce314' + ) // 'DNS Zone Contributor' + principalId: managedCluster.properties.ingressProfile.webAppRouting.identity.objectId + principalType: 'ServicePrincipal' + } + scope: dnsZone +} + +@description('The resource ID of the managed cluster.') +output resourceId string = managedCluster.id + +@description('The resource group the managed cluster was deployed into.') +output resourceGroupName string = resourceGroup().name + +@description('The name of the managed cluster.') +output name string = managedCluster.name + +@description('The control plane FQDN of the managed cluster.') +output controlPlaneFQDN string = enablePrivateCluster + ? managedCluster.properties.privateFQDN + : managedCluster.properties.fqdn + +@description('The principal ID of the system assigned identity.') +output systemAssignedMIPrincipalId string = managedCluster.?identity.?principalId ?? '' + +@description('The Client ID of the AKS identity.') +output kubeletIdentityClientId string = managedCluster.properties.?identityProfile.?kubeletidentity.?clientId ?? '' + +@description('The Object ID of the AKS identity.') +output kubeletIdentityObjectId string = managedCluster.properties.?identityProfile.?kubeletidentity.?objectId ?? '' + +@description('The Resource ID of the AKS identity.') +output kubeletIdentityResourceId string = managedCluster.properties.?identityProfile.?kubeletidentity.?resourceId ?? '' + +@description('The Object ID of the OMS agent identity.') +output omsagentIdentityObjectId string = managedCluster.properties.?addonProfiles.?omsagent.?identity.?objectId ?? '' + +@description('The Object ID of the Key Vault Secrets Provider identity.') +output keyvaultIdentityObjectId string = managedCluster.properties.?addonProfiles.?azureKeyvaultSecretsProvider.?identity.?objectId ?? '' + +@description('The Client ID of the Key Vault Secrets Provider identity.') +output keyvaultIdentityClientId string = managedCluster.properties.?addonProfiles.?azureKeyvaultSecretsProvider.?identity.?clientId ?? '' + +@description('The Object ID of Application Gateway Ingress Controller (AGIC) identity.') +output ingressApplicationGatewayIdentityObjectId string = managedCluster.properties.?addonProfiles.?ingressApplicationGateway.?identity.?objectId ?? '' + +@description('The location the resource was deployed into.') +output location string = managedCluster.location + +@description('The OIDC token issuer URL.') +output oidcIssuerUrl string = managedCluster.properties.?oidcIssuerProfile.?issuerURL ?? '' + +@description('The addonProfiles of the Kubernetes cluster.') +output addonProfiles object = managedCluster.properties.?addonProfiles ?? {} + +@description('The Object ID of Web Application Routing.') +output webAppRoutingIdentityObjectId string = managedCluster.properties.?ingressProfile.?webAppRouting.?identity.?objectId ?? '' + +// @description('The public IP address of the managed cluster.') +// output outboundIpResourceId string = managedCluster.properties.networkProfile.natGatewayProfile.effectiveOutboundIPs[0].id + +// =============== // +// Definitions // +// =============== // + +type agentPoolType = { + @description('Required. The name of the agent pool.') + name: string? + + @description('Optional. The availability zones of the agent pool.') + availabilityZones: string[]? + + @description('Optional. The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive).') + count: int? + + @description('Optional. The source resource ID to create the agent pool from.') + sourceResourceId: string? + + @description('Optional. Whether to enable auto-scaling for the agent pool.') + enableAutoScaling: bool? + + @description('Optional. Whether to enable encryption at host for the agent pool.') + enableEncryptionAtHost: bool? + + @description('Optional. Whether to enable FIPS for the agent pool.') + enableFIPS: bool? + + @description('Optional. Whether to enable node public IP for the agent pool.') + enableNodePublicIP: bool? + + @description('Optional. Whether to enable Ultra SSD for the agent pool.') + enableUltraSSD: bool? + + @description('Optional. The GPU instance profile of the agent pool.') + gpuInstanceProfile: ('MIG1g' | 'MIG2g' | 'MIG3g' | 'MIG4g' | 'MIG7g')? + + @description('Optional. The kubelet disk type of the agent pool.') + kubeletDiskType: string? + + @description('Optional. The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive).') + maxCount: int? + + @description('Optional. The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive).') + minCount: int? + + @description('Optional. The maximum number of pods that can run on a node.') + maxPods: int? + + @description('Optional. The minimum number of pods that can run on a node.') + minPods: int? + + @description('Optional. The mode of the agent pool.') + mode: ('System' | 'User')? + + @description('Optional. The node labels of the agent pool.') + nodeLabels: object? + + @description('Optional. The node public IP prefix ID of the agent pool.') + nodePublicIpPrefixId: string? + + @description('Optional. The node taints of the agent pool.') + nodeTaints: string[]? + + @description('Optional. The Kubernetes version of the agent pool.') + orchestratorVersion: string? + + @description('Optional. The OS disk size in GB of the agent pool.') + osDiskSizeGB: int? + + @description('Optional. The OS disk type of the agent pool.') + osDiskType: string? + + @description('Optional. The OS SKU of the agent pool.') + osSku: string? + + @description('Optional. The OS type of the agent pool.') + osType: ('Linux' | 'Windows')? + + @description('Optional. The pod subnet ID of the agent pool.') + podSubnetId: string? + + @description('Optional. The proximity placement group resource ID of the agent pool.') + proximityPlacementGroupResourceId: string? + + @description('Optional. The scale down mode of the agent pool.') + scaleDownMode: ('Delete' | 'Deallocate')? + + @description('Optional. The scale set eviction policy of the agent pool.') + scaleSetEvictionPolicy: ('Delete' | 'Deallocate')? + + @description('Optional. The scale set priority of the agent pool.') + scaleSetPriority: ('Low' | 'Regular' | 'Spot')? + + @description('Optional. The spot max price of the agent pool.') + spotMaxPrice: int? + + @description('Optional. Whether to enable SSH access to the nodes.') + sshAccess: ('LocalUser' | 'Disabled')? + + @description('Optional. The tags of the agent pool.') + tags: object? + + @description('Optional. The type of the agent pool.') + type: ('AvailabilitySet' | 'VirtualMachineScaleSets')? + + @description('Optional. The maximum number of nodes that can be created during an upgrade.') + maxSurge: string? + + @description('Optional. The VM size of the agent pool.') + vmSize: string? + + @description('Optional. The VNet subnet ID of the agent pool.') + vnetSubnetID: string? + + @description('Optional. The workload runtime of the agent pool.') + workloadRuntime: string? + + @description('Optional. The enable default telemetry of the agent pool.') + enableDefaultTelemetry: bool? +}[]? + +type managedIdentitiesType = { + @description('Optional. Enables system assigned managed identity on the resource.') + systemAssigned: bool? + + @description('Optional. The resource ID(s) to assign to the resource.') + userAssignedResourcesIds: string[]? +}? + +type lockType = { + @description('Optional. Specify the name of lock.') + name: string? + + @description('Optional. Specify the type of lock.') + kind: ('CanNotDelete' | 'ReadOnly' | 'None')? +}? + +type roleAssignmentType = { + @description('Optional. The name (as GUID) of the role assignment. If not provided, a GUID will be generated.') + name: string? + + @description('Required. The role to assign. You can provide either the display name of the role definition, the role definition GUID, or its fully qualified ID in the following format: \'/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11\'.') + roleDefinitionIdOrName: string + + @description('Required. The principal ID of the principal (user/group/identity) to assign the role to.') + principalId: string + + @description('Optional. The principal type of the assigned principal ID.') + principalType: ('ServicePrincipal' | 'Group' | 'User' | 'ForeignGroup' | 'Device')? + + @description('Optional. The description of the role assignment.') + description: string? + + @description('Optional. The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase "foo_storage_container".') + condition: string? + + @description('Optional. Version of the condition.') + conditionVersion: '2.0'? + + @description('Optional. The Resource Id of the delegated managed identity resource.') + delegatedManagedIdentityResourceId: string? +}[]? + +type diagnosticSettingType = { + @description('Optional. The name of diagnostic setting.') + name: string? + + @description('Optional. The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to `[]` to disable log collection.') + logCategoriesAndGroups: { + @description('Optional. Name of a Diagnostic Log category for a resource type this setting is applied to. Set the specific logs to collect here.') + category: string? + + @description('Optional. Name of a Diagnostic Log category group for a resource type this setting is applied to. Set to `allLogs` to collect all logs.') + categoryGroup: string? + + @description('Optional. Enable or disable the category explicitly. Default is `true`.') + enabled: bool? + }[]? + + @description('Optional. The name of metrics that will be streamed. "allMetrics" includes all possible metrics for the resource. Set to `[]` to disable metric collection.') + metricCategories: { + @description('Required. Name of a Diagnostic Metric category for a resource type this setting is applied to. Set to `AllMetrics` to collect all metrics.') + category: string + + @description('Optional. Enable or disable the category explicitly. Default is `true`.') + enabled: bool? + }[]? + + @description('Optional. A string indicating whether the export to Log Analytics should use the default destination type, i.e. AzureDiagnostics, or use a destination type.') + logAnalyticsDestinationType: ('Dedicated' | 'AzureDiagnostics' | null)? + + @description('Optional. Resource ID of the diagnostic log analytics workspace. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub.') + workspaceResourceId: string? + + @description('Optional. Resource ID of the diagnostic storage account. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub.') + storageAccountResourceId: string? + + @description('Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to.') + eventHubAuthorizationRuleResourceId: string? + + @description('Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub.') + eventHubName: string? + + @description('Optional. The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs.') + marketplacePartnerResourceId: string? +}[]? + +type fluxConfigurationProtectedSettingsType = { + @description('Optional. The SSH private key to use for Git authentication.') + sshPrivateKey: string? +}? + +type extensionType = { + @description('Required. The name of the extension.') + name: string? + + @description('Optional. Namespace where the extension Release must be placed.') + releaseNamespace: string? + + @description('Optional. Namespace where the extension will be created for an Namespace scoped extension.') + targetNamespace: string? + + @description('Required. The release train of the extension.') + releaseTrain: string? + + @description('Optional. The configuration protected settings of the extension.') + configurationProtectedSettings: fluxConfigurationProtectedSettingsType? + + @description('Optional. The configuration settings of the extension.') + configurationSettings: object? + + @description('Optional. The version of the extension.') + version: string? + + @description('Optional. The flux configurations of the extension.') + configurations: array? +}? + +type customerManagedKeyType = { + @description('Required. The resource ID of a key vault to reference a customer managed key for encryption from.') + keyVaultResourceId: string + + @description('Required. The name of the customer managed key to use for encryption.') + keyName: string + + @description('Optional. The version of the customer managed key to reference for encryption. If not provided, using \'latest\'.') + keyVersion: string? + + @description('Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public.') + keyVaultNetworkAccess: ('Private' | 'Public') +}? + +type maintenanceConfigurationType = { + @description('Required. Maintenance window for the maintenance configuration.') + maintenanceWindow: object +}? diff --git a/bicep/modules/managed-cluster/main.json b/bicep/modules/managed-cluster/main.json new file mode 100644 index 00000000..5e117837 --- /dev/null +++ b/bicep/modules/managed-cluster/main.json @@ -0,0 +1,2980 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.29.47.4906", + "templateHash": "6707709888421096485" + }, + "name": "Azure Kubernetes Service (AKS) Managed Clusters", + "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", + "owner": "Azure/module-maintainers" + }, + "definitions": { + "agentPoolType": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Required. The name of the agent pool." + } + }, + "availabilityZones": { + "type": "array", + "items": { + "type": "string" + }, + "nullable": true, + "metadata": { + "description": "Optional. The availability zones of the agent pool." + } + }, + "count": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "sourceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The source resource ID to create the agent pool from." + } + }, + "enableAutoScaling": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable auto-scaling for the agent pool." + } + }, + "enableEncryptionAtHost": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable encryption at host for the agent pool." + } + }, + "enableFIPS": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable FIPS for the agent pool." + } + }, + "enableNodePublicIP": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable node public IP for the agent pool." + } + }, + "enableUltraSSD": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable Ultra SSD for the agent pool." + } + }, + "gpuInstanceProfile": { + "type": "string", + "allowedValues": [ + "MIG1g", + "MIG2g", + "MIG3g", + "MIG4g", + "MIG7g" + ], + "nullable": true, + "metadata": { + "description": "Optional. The GPU instance profile of the agent pool." + } + }, + "kubeletDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The kubelet disk type of the agent pool." + } + }, + "maxCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "minCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "maxPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of pods that can run on a node." + } + }, + "minPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of pods that can run on a node." + } + }, + "mode": { + "type": "string", + "allowedValues": [ + "System", + "User" + ], + "nullable": true, + "metadata": { + "description": "Optional. The mode of the agent pool." + } + }, + "nodeLabels": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The node labels of the agent pool." + } + }, + "nodePublicIpPrefixId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The node public IP prefix ID of the agent pool." + } + }, + "nodeTaints": { + "type": "array", + "items": { + "type": "string" + }, + "nullable": true, + "metadata": { + "description": "Optional. The node taints of the agent pool." + } + }, + "orchestratorVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The Kubernetes version of the agent pool." + } + }, + "osDiskSizeGB": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The OS disk size in GB of the agent pool." + } + }, + "osDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The OS disk type of the agent pool." + } + }, + "osSku": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The OS SKU of the agent pool." + } + }, + "osType": { + "type": "string", + "allowedValues": [ + "Linux", + "Windows" + ], + "nullable": true, + "metadata": { + "description": "Optional. The OS type of the agent pool." + } + }, + "podSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The pod subnet ID of the agent pool." + } + }, + "proximityPlacementGroupResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The proximity placement group resource ID of the agent pool." + } + }, + "scaleDownMode": { + "type": "string", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale down mode of the agent pool." + } + }, + "scaleSetEvictionPolicy": { + "type": "string", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale set eviction policy of the agent pool." + } + }, + "scaleSetPriority": { + "type": "string", + "allowedValues": [ + "Low", + "Regular", + "Spot" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale set priority of the agent pool." + } + }, + "spotMaxPrice": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The spot max price of the agent pool." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The tags of the agent pool." + } + }, + "type": { + "type": "string", + "allowedValues": [ + "AvailabilitySet", + "VirtualMachineScaleSets" + ], + "nullable": true, + "metadata": { + "description": "Optional. The type of the agent pool." + } + }, + "maxSurge": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of nodes that can be created during an upgrade." + } + }, + "vmSize": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The VM size of the agent pool." + } + }, + "vnetSubnetID": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The VNet subnet ID of the agent pool." + } + }, + "workloadRuntime": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The workload runtime of the agent pool." + } + }, + "enableDefaultTelemetry": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. The enable default telemetry of the agent pool." + } + } + } + }, + "nullable": true + }, + "managedIdentitiesType": { + "type": "object", + "properties": { + "systemAssigned": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Enables system assigned managed identity on the resource." + } + }, + "userAssignedResourcesIds": { + "type": "array", + "items": { + "type": "string" + }, + "nullable": true, + "metadata": { + "description": "Optional. The resource ID(s) to assign to the resource." + } + } + }, + "nullable": true + }, + "lockType": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specify the name of lock." + } + }, + "kind": { + "type": "string", + "allowedValues": [ + "CanNotDelete", + "None", + "ReadOnly" + ], + "nullable": true, + "metadata": { + "description": "Optional. Specify the type of lock." + } + } + }, + "nullable": true + }, + "roleAssignmentType": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The name (as GUID) of the role assignment. If not provided, a GUID will be generated." + } + }, + "roleDefinitionIdOrName": { + "type": "string", + "metadata": { + "description": "Required. The role to assign. You can provide either the display name of the role definition, the role definition GUID, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'." + } + }, + "principalId": { + "type": "string", + "metadata": { + "description": "Required. The principal ID of the principal (user/group/identity) to assign the role to." + } + }, + "principalType": { + "type": "string", + "allowedValues": [ + "Device", + "ForeignGroup", + "Group", + "ServicePrincipal", + "User" + ], + "nullable": true, + "metadata": { + "description": "Optional. The principal type of the assigned principal ID." + } + }, + "description": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The description of the role assignment." + } + }, + "condition": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase \"foo_storage_container\"." + } + }, + "conditionVersion": { + "type": "string", + "allowedValues": [ + "2.0" + ], + "nullable": true, + "metadata": { + "description": "Optional. Version of the condition." + } + }, + "delegatedManagedIdentityResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The Resource Id of the delegated managed identity resource." + } + } + } + }, + "nullable": true + }, + "diagnosticSettingType": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The name of diagnostic setting." + } + }, + "logCategoriesAndGroups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "category": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Name of a Diagnostic Log category for a resource type this setting is applied to. Set the specific logs to collect here." + } + }, + "categoryGroup": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Name of a Diagnostic Log category group for a resource type this setting is applied to. Set to `allLogs` to collect all logs." + } + }, + "enabled": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Enable or disable the category explicitly. Default is `true`." + } + } + } + }, + "nullable": true, + "metadata": { + "description": "Optional. The name of logs that will be streamed. \"allLogs\" includes all possible logs for the resource. Set to `[]` to disable log collection." + } + }, + "metricCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "category": { + "type": "string", + "metadata": { + "description": "Required. Name of a Diagnostic Metric category for a resource type this setting is applied to. Set to `AllMetrics` to collect all metrics." + } + }, + "enabled": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Enable or disable the category explicitly. Default is `true`." + } + } + } + }, + "nullable": true, + "metadata": { + "description": "Optional. The name of metrics that will be streamed. \"allMetrics\" includes all possible metrics for the resource. Set to `[]` to disable metric collection." + } + }, + "logAnalyticsDestinationType": { + "type": "string", + "allowedValues": [ + "AzureDiagnostics", + "Dedicated" + ], + "nullable": true, + "metadata": { + "description": "Optional. A string indicating whether the export to Log Analytics should use the default destination type, i.e. AzureDiagnostics, or use a destination type." + } + }, + "workspaceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Resource ID of the diagnostic log analytics workspace. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub." + } + }, + "storageAccountResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Resource ID of the diagnostic storage account. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub." + } + }, + "eventHubAuthorizationRuleResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to." + } + }, + "eventHubName": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub." + } + }, + "marketplacePartnerResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs." + } + } + } + }, + "nullable": true + }, + "fluxConfigurationProtectedSettingsType": { + "type": "object", + "properties": { + "sshPrivateKey": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The SSH private key to use for Git authentication." + } + } + }, + "nullable": true + }, + "extensionType": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Required. The name of the extension." + } + }, + "releaseNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension Release must be placed." + } + }, + "targetNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension will be created for an Namespace scoped extension." + } + }, + "releaseTrain": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Required. The release train of the extension." + } + }, + "configurationProtectedSettings": { + "$ref": "#/definitions/fluxConfigurationProtectedSettingsType", + "nullable": true, + "metadata": { + "description": "Optional. The configuration protected settings of the extension." + } + }, + "configurationSettings": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The configuration settings of the extension." + } + }, + "version": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The version of the extension." + } + }, + "configurations": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The flux configurations of the extension." + } + } + }, + "nullable": true + }, + "customerManagedKeyType": { + "type": "object", + "properties": { + "keyVaultResourceId": { + "type": "string", + "metadata": { + "description": "Required. The resource ID of a key vault to reference a customer managed key for encryption from." + } + }, + "keyName": { + "type": "string", + "metadata": { + "description": "Required. The name of the customer managed key to use for encryption." + } + }, + "keyVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The version of the customer managed key to reference for encryption. If not provided, using 'latest'." + } + }, + "keyVaultNetworkAccess": { + "type": "string", + "allowedValues": [ + "Private", + "Public" + ], + "metadata": { + "description": "Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public." + } + } + }, + "nullable": true + }, + "maintenanceConfigurationType": { + "type": "object", + "properties": { + "maintenanceWindow": { + "type": "object", + "metadata": { + "description": "Required. Maintenance window for the maintenance configuration." + } + } + }, + "nullable": true + } + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. Specifies the name of the AKS cluster." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Specifies the location of AKS cluster. It picks up Resource Group's location by default." + } + }, + "dnsPrefix": { + "type": "string", + "defaultValue": "[parameters('name')]", + "metadata": { + "description": "Optional. Specifies the DNS prefix specified when creating the managed cluster." + } + }, + "managedIdentities": { + "$ref": "#/definitions/managedIdentitiesType", + "metadata": { + "description": "Optional. The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both." + } + }, + "networkDataplane": { + "type": "string", + "nullable": true, + "allowedValues": [ + "azure", + "cilium" + ], + "metadata": { + "description": "Optional. Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin." + } + }, + "networkPlugin": { + "type": "string", + "nullable": true, + "allowedValues": [ + "azure", + "kubenet" + ], + "metadata": { + "description": "Optional. Specifies the network plugin used for building Kubernetes network." + } + }, + "networkPluginMode": { + "type": "string", + "nullable": true, + "allowedValues": [ + "overlay" + ], + "metadata": { + "description": "Optional. Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin." + } + }, + "networkPolicy": { + "type": "string", + "nullable": true, + "allowedValues": [ + "azure", + "calico" + ], + "metadata": { + "description": "Optional. Specifies the network policy used for building Kubernetes network. - calico or azure." + } + }, + "podCidr": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the CIDR notation IP range from which to assign pod IPs when kubenet is used." + } + }, + "serviceCidr": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. A CIDR notation IP range from which to assign service cluster IPs. It must not overlap with any Subnet IP ranges." + } + }, + "dnsServiceIP": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr." + } + }, + "loadBalancerSku": { + "type": "string", + "defaultValue": "standard", + "allowedValues": [ + "basic", + "standard" + ], + "metadata": { + "description": "Optional. Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools." + } + }, + "managedOutboundIPCount": { + "type": "int", + "defaultValue": 0, + "metadata": { + "description": "Optional. Outbound IP Count for the Load balancer." + } + }, + "backendPoolType": { + "type": "string", + "defaultValue": "NodeIPConfiguration", + "allowedValues": [ + "NodeIP", + "NodeIPConfiguration" + ], + "metadata": { + "description": "Optional. The type of the managed inbound Load Balancer BackendPool." + } + }, + "outboundType": { + "type": "string", + "defaultValue": "loadBalancer", + "allowedValues": [ + "loadBalancer", + "userDefinedRouting", + "managedNATGateway", + "userAssignedNATGateway" + ], + "metadata": { + "description": "Optional. Specifies outbound (egress) routing method." + } + }, + "skuTier": { + "type": "string", + "defaultValue": "Standard", + "allowedValues": [ + "Free", + "Premium", + "Standard" + ], + "metadata": { + "description": "Optional. Tier of a managed cluster SKU." + } + }, + "kubernetesVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Version of Kubernetes specified when creating the managed cluster." + } + }, + "adminUsername": { + "type": "string", + "defaultValue": "azureuser", + "metadata": { + "description": "Optional. Specifies the administrator username of Linux virtual machines." + } + }, + "sshPublicKey": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the SSH RSA public key string for the Linux nodes." + } + }, + "aksServicePrincipalProfile": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster." + } + }, + "aadProfileClientAppID": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The client AAD application ID." + } + }, + "aadProfileServerAppID": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The server AAD application ID." + } + }, + "aadProfileServerAppSecret": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The server AAD application secret." + } + }, + "aadProfileTenantId": { + "type": "string", + "defaultValue": "[subscription().tenantId]", + "metadata": { + "description": "Optional. Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication." + } + }, + "aadProfileAdminGroupObjectIDs": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the AAD group object IDs that will have admin role of the cluster." + } + }, + "aadProfileManaged": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies whether to enable managed AAD integration." + } + }, + "enableRBAC": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Whether to enable Kubernetes Role-Based Access Control." + } + }, + "aadProfileEnableAzureRBAC": { + "type": "bool", + "defaultValue": "[parameters('enableRBAC')]", + "metadata": { + "description": "Optional. Specifies whether to enable Azure RBAC for Kubernetes authorization." + } + }, + "disableLocalAccounts": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled." + } + }, + "nodeResourceGroup": { + "type": "string", + "defaultValue": "[format('{0}_aks_{1}_nodes', resourceGroup().name, parameters('name'))]", + "metadata": { + "description": "Optional. Name of the resource group containing agent pool nodes." + } + }, + "authorizedIPRanges": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer." + } + }, + "disableRunCommand": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to disable run command for the cluster or not." + } + }, + "publicNetworkAccess": { + "type": "string", + "defaultValue": "Disabled", + "allowedValues": [ + "Enabled", + "Disabled", + "SecuredByPerimeter" + ], + "metadata": { + "description": "Optional. Allow or deny public network access for AKS." + } + }, + "enablePrivateCluster": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether to create the cluster as a private cluster or not." + } + }, + "enablePrivateClusterPublicFQDN": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to create additional public FQDN for private cluster or not." + } + }, + "privateDNSZone": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Private DNS Zone configuration. Set to 'system' and AKS will create a private DNS zone in the node resource group. Set to '' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone." + } + }, + "primaryAgentPoolProfile": { + "type": "array", + "metadata": { + "description": "Required. Properties of the primary agent pool." + } + }, + "agentPools": { + "$ref": "#/definitions/agentPoolType", + "metadata": { + "description": "Optional. Define one or more secondary/additional agent pools." + } + }, + "maintenanceConfiguration": { + "$ref": "#/definitions/maintenanceConfigurationType", + "metadata": { + "description": "Optional. Whether or not to use AKS Automatic mode." + } + }, + "costAnalysisEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the cost analysis add-on is enabled or not. If Enabled `enableStorageProfileDiskCSIDriver` is set to true as it is needed." + } + }, + "httpApplicationRoutingEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the httpApplicationRouting add-on is enabled or not." + } + }, + "webApplicationRoutingEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the webApplicationRoutingEnabled add-on is enabled or not." + } + }, + "dnsZoneResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`." + } + }, + "enableDnsZoneContributorRoleAssignment": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided." + } + }, + "ingressApplicationGatewayEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not." + } + }, + "appGatewayResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Conditional. Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`." + } + }, + "aciConnectorLinuxEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the aciConnectorLinux add-on is enabled or not." + } + }, + "azurePolicyEnabled": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled." + } + }, + "openServiceMeshEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the openServiceMesh add-on is enabled or not." + } + }, + "azurePolicyVersion": { + "type": "string", + "defaultValue": "v2", + "metadata": { + "description": "Optional. Specifies the azure policy version to use." + } + }, + "kubeDashboardEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the kubeDashboard add-on is enabled or not." + } + }, + "enableKeyvaultSecretsProvider": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the KeyvaultSecretsProvider add-on is enabled or not." + } + }, + "enableSecretRotation": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation." + } + }, + "autoScalerProfileScanInterval": { + "type": "string", + "defaultValue": "10s", + "metadata": { + "description": "Optional. Specifies the scan interval of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownDelayAfterAdd": { + "type": "string", + "defaultValue": "10m", + "metadata": { + "description": "Optional. Specifies the scale down delay after add of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownDelayAfterDelete": { + "type": "string", + "defaultValue": "20s", + "metadata": { + "description": "Optional. Specifies the scale down delay after delete of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownDelayAfterFailure": { + "type": "string", + "defaultValue": "3m", + "metadata": { + "description": "Optional. Specifies scale down delay after failure of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownUnneededTime": { + "type": "string", + "defaultValue": "10m", + "metadata": { + "description": "Optional. Specifies the scale down unneeded time of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownUnreadyTime": { + "type": "string", + "defaultValue": "20m", + "metadata": { + "description": "Optional. Specifies the scale down unready time of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileUtilizationThreshold": { + "type": "string", + "defaultValue": "0.5", + "metadata": { + "description": "Optional. Specifies the utilization threshold of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileMaxGracefulTerminationSec": { + "type": "string", + "defaultValue": "600", + "metadata": { + "description": "Optional. Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileBalanceSimilarNodeGroups": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies the balance of similar node groups for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileExpander": { + "type": "string", + "defaultValue": "random", + "allowedValues": [ + "least-waste", + "most-pods", + "priority", + "random" + ], + "metadata": { + "description": "Optional. Specifies the expand strategy for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileMaxEmptyBulkDelete": { + "type": "string", + "defaultValue": "10", + "metadata": { + "description": "Optional. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileMaxNodeProvisionTime": { + "type": "string", + "defaultValue": "15m", + "metadata": { + "description": "Optional. Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an \"m\". No unit of time other than minutes (m) is supported." + } + }, + "autoScalerProfileMaxTotalUnreadyPercentage": { + "type": "string", + "defaultValue": "45", + "metadata": { + "description": "Optional. Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0." + } + }, + "autoScalerProfileNewPodScaleUpDelay": { + "type": "string", + "defaultValue": "0s", + "metadata": { + "description": "Optional. For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit (\"s\" for seconds, \"m\" for minutes, \"h\" for hours, etc)." + } + }, + "autoScalerProfileOkTotalUnreadyCount": { + "type": "string", + "defaultValue": "3", + "metadata": { + "description": "Optional. Specifies the OK total unready count for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileSkipNodesWithLocalStorage": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileSkipNodesWithSystemPods": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster." + } + }, + "autoUpgradeProfileUpgradeChannel": { + "type": "string", + "defaultValue": "stable", + "allowedValues": [ + "node-image", + "none", + "patch", + "rapid", + "stable" + ], + "metadata": { + "description": "Optional. Auto-upgrade channel on the AKS cluster." + } + }, + "podIdentityProfileAllowNetworkPluginKubenet": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing." + } + }, + "podIdentityProfileEnable": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the pod identity addon is enabled." + } + }, + "podIdentityProfileUserAssignedIdentities": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The pod identities to use in the cluster." + } + }, + "podIdentityProfileUserAssignedIdentityExceptions": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The pod identity exceptions to allow." + } + }, + "enableOidcIssuerProfile": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the The OIDC issuer profile of the Managed Cluster is enabled." + } + }, + "enableWorkloadIdentity": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Workload Identity. Requires OIDC issuer profile to be enabled." + } + }, + "enableAzureDefender": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Azure Defender." + } + }, + "enableImageCleaner": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Image Cleaner for Kubernetes." + } + }, + "imageCleanerIntervalHours": { + "type": "int", + "defaultValue": 24, + "minValue": 24, + "metadata": { + "description": "Optional. The interval in hours Image Cleaner will run. The maximum value is three months." + } + }, + "enablePodSecurityPolicy": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Kubernetes pod security policy. Requires enabling the pod security policy feature flag on the subscription." + } + }, + "enableStorageProfileBlobCSIDriver": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the AzureBlob CSI Driver for the storage profile is enabled." + } + }, + "enableStorageProfileDiskCSIDriver": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the AzureDisk CSI Driver for the storage profile is enabled." + } + }, + "enableStorageProfileFileCSIDriver": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the AzureFile CSI Driver for the storage profile is enabled." + } + }, + "enableStorageProfileSnapshotController": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the snapshot controller for the storage profile is enabled." + } + }, + "supportPlan": { + "type": "string", + "defaultValue": "KubernetesOfficial", + "allowedValues": [ + "AKSLongTermSupport", + "KubernetesOfficial" + ], + "metadata": { + "description": "Optional. The support plan for the Managed Cluster." + } + }, + "diagnosticSettings": { + "$ref": "#/definitions/diagnosticSettingType", + "metadata": { + "description": "Optional. The diagnostic settings of the service." + } + }, + "omsAgentEnabled": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies whether the OMS agent is enabled." + } + }, + "monitoringWorkspaceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Resource ID of the monitoring log analytics workspace." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "roleAssignments": { + "$ref": "#/definitions/roleAssignmentType", + "metadata": { + "description": "Optional. Array of role assignments to create." + } + }, + "lock": { + "$ref": "#/definitions/lockType", + "metadata": { + "description": "Optional. The lock settings of the service." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Tags of the resource." + } + }, + "diskEncryptionSetResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided." + } + }, + "fluxExtension": { + "$ref": "#/definitions/extensionType", + "metadata": { + "description": "Optional. Settings and configurations for the flux extension." + } + }, + "httpProxyConfig": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Configurations for provisioning the cluster with HTTP proxy servers." + } + }, + "identityProfile": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Identities associated with the cluster." + } + }, + "kedaAddon": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Enables Kubernetes Event-driven Autoscaling (KEDA)." + } + }, + "vpaAddon": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable VPA add-on in cluster. Default value is false." + } + }, + "customerManagedKey": { + "$ref": "#/definitions/customerManagedKeyType", + "metadata": { + "description": "Optional. The customer managed key definition." + } + }, + "enableAzureMonitorProfileMetrics": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the metric state of the kubenetes cluster is enabled." + } + }, + "enableContainerInsights": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Indicates if Azure Monitor Container Insights Logs Addon is enabled." + } + }, + "disableCustomMetrics": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Indicates whether custom metrics collection has to be disabled or not. If not specified the default is false. No custom metrics will be emitted if this field is false but the container insights enabled field is false." + } + }, + "disablePrometheusMetricsScraping": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Indicates whether prometheus metrics scraping is disabled or not. If not specified the default is false. No prometheus metrics will be emitted if this field is false but the container insights enabled field is false." + } + }, + "syslogPort": { + "type": "int", + "defaultValue": 28330, + "metadata": { + "description": "Optional. The syslog host port. If not specified, the default port is 28330." + } + }, + "metricLabelsAllowlist": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional. A comma-separated list of kubernetes cluster metrics labels." + } + }, + "metricAnnotationsAllowList": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional. A comma-separated list of Kubernetes cluster metrics annotations." + } + } + }, + "variables": { + "copy": [ + { + "name": "formattedRoleAssignments", + "count": "[length(coalesce(parameters('roleAssignments'), createArray()))]", + "input": "[union(coalesce(parameters('roleAssignments'), createArray())[copyIndex('formattedRoleAssignments')], createObject('roleDefinitionId', coalesce(tryGet(variables('builtInRoleNames'), coalesce(parameters('roleAssignments'), createArray())[copyIndex('formattedRoleAssignments')].roleDefinitionIdOrName), if(contains(coalesce(parameters('roleAssignments'), createArray())[copyIndex('formattedRoleAssignments')].roleDefinitionIdOrName, '/providers/Microsoft.Authorization/roleDefinitions/'), coalesce(parameters('roleAssignments'), createArray())[copyIndex('formattedRoleAssignments')].roleDefinitionIdOrName, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', coalesce(parameters('roleAssignments'), createArray())[copyIndex('formattedRoleAssignments')].roleDefinitionIdOrName)))))]" + } + ], + "formattedUserAssignedIdentities": "[reduce(map(coalesce(tryGet(parameters('managedIdentities'), 'userAssignedResourcesIds'), createArray()), lambda('id', createObject(format('{0}', lambdaVariables('id')), createObject()))), createObject(), lambda('cur', 'next', union(lambdaVariables('cur'), lambdaVariables('next'))))]", + "identity": "[if(not(empty(parameters('managedIdentities'))), createObject('type', if(coalesce(tryGet(parameters('managedIdentities'), 'systemAssigned'), false()), 'SystemAssigned', if(not(empty(coalesce(tryGet(parameters('managedIdentities'), 'userAssignedResourcesIds'), createObject()))), 'UserAssigned', null())), 'userAssignedIdentities', if(not(empty(variables('formattedUserAssignedIdentities'))), variables('formattedUserAssignedIdentities'), null())), null())]", + "builtInRoleNames": { + "Azure Kubernetes Fleet Manager Contributor Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '63bb64ad-9799-4770-b5c3-24ed299a07bf')]", + "Azure Kubernetes Fleet Manager RBAC Admin": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '434fb43a-c01c-447e-9f67-c3ad923cfaba')]", + "Azure Kubernetes Fleet Manager RBAC Cluster Admin": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18ab4d3d-a1bf-4477-8ad9-8359bc988f69')]", + "Azure Kubernetes Fleet Manager RBAC Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '30b27cfc-9c84-438e-b0ce-70e35255df80')]", + "Azure Kubernetes Fleet Manager RBAC Writer": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5af6afb3-c06c-4fa4-8848-71a8aee05683')]", + "Azure Kubernetes Service Cluster Admin Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8')]", + "Azure Kubernetes Service Cluster Monitoring User": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '1afdec4b-e479-420e-99e7-f82237c7c5e6')]", + "Azure Kubernetes Service Cluster User Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4abbcc35-e782-43d8-92c5-2d3f1bd2253f')]", + "Azure Kubernetes Service Contributor Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ed7f3fbd-7b88-4dd4-9017-9adb7ce333f8')]", + "Azure Kubernetes Service RBAC Admin": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3498e952-d568-435e-9b2c-8d77e338d7f7')]", + "Azure Kubernetes Service RBAC Cluster Admin": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b')]", + "Azure Kubernetes Service RBAC Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f6c6a51-bcf8-42ba-9220-52d62157d7db')]", + "Azure Kubernetes Service RBAC Writer": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a7ffa36f-339b-4b5c-8bdf-e2c188b2c0eb')]", + "Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", + "Kubernetes Agentless Operator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'd5a2ae44-610b-4500-93be-660a0c5f5ca6')]", + "Owner": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", + "Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]", + "Role Based Access Control Administrator (Preview)": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f58310d9-a9f6-439a-9e8d-f62e7b41a168')]", + "User Access Administrator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9')]" + } + }, + "resources": { + "cMKKeyVault::cMKKey": { + "condition": "[and(not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'))), and(not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'))), not(empty(tryGet(parameters('customerManagedKey'), 'keyName')))))]", + "existing": true, + "type": "Microsoft.KeyVault/vaults/keys", + "apiVersion": "2023-02-01", + "subscriptionId": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2]]", + "resourceGroup": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]]", + "name": "[format('{0}/{1}', last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/')), coalesce(tryGet(parameters('customerManagedKey'), 'keyName'), 'dummyKey'))]", + "dependsOn": [ + "cMKKeyVault" + ] + }, + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2024-03-01", + "name": "[format('46d3xbcp.res.containerservice-managedcluster.{0}.{1}', replace('-..--..-', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "cMKKeyVault": { + "condition": "[not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId')))]", + "existing": true, + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2023-02-01", + "subscriptionId": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2]]", + "resourceGroup": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]]", + "name": "[last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/'))]" + }, + "managedCluster": { + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2024-03-02-preview", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "identity": "[variables('identity')]", + "sku": { + "name": "Base", + "tier": "[parameters('skuTier')]" + }, + "properties": { + "httpProxyConfig": "[parameters('httpProxyConfig')]", + "identityProfile": "[parameters('identityProfile')]", + "diskEncryptionSetID": "[parameters('diskEncryptionSetResourceId')]", + "kubernetesVersion": "[parameters('kubernetesVersion')]", + "dnsPrefix": "[parameters('dnsPrefix')]", + "agentPoolProfiles": "[parameters('primaryAgentPoolProfile')]", + "linuxProfile": "[if(not(empty(parameters('sshPublicKey'))), createObject('adminUsername', parameters('adminUsername'), 'ssh', createObject('publicKeys', createArray(createObject('keyData', coalesce(parameters('sshPublicKey'), ''))))), null())]", + "servicePrincipalProfile": "[parameters('aksServicePrincipalProfile')]", + "metricsProfile": { + "costAnalysis": { + "enabled": "[if(equals(parameters('skuTier'), 'free'), false(), parameters('costAnalysisEnabled'))]" + } + }, + "ingressProfile": { + "webAppRouting": { + "enabled": "[parameters('webApplicationRoutingEnabled')]", + "dnsZoneResourceIds": "[if(not(empty(parameters('dnsZoneResourceId'))), createArray(parameters('dnsZoneResourceId')), null())]" + } + }, + "addonProfiles": { + "httpApplicationRouting": { + "enabled": "[parameters('httpApplicationRoutingEnabled')]" + }, + "ingressApplicationGateway": { + "enabled": "[and(parameters('ingressApplicationGatewayEnabled'), not(empty(parameters('appGatewayResourceId'))))]", + "config": "[if(and(parameters('ingressApplicationGatewayEnabled'), not(empty(parameters('appGatewayResourceId')))), createObject('applicationGatewayId', parameters('appGatewayResourceId'), 'effectiveApplicationGatewayId', parameters('appGatewayResourceId')), null())]" + }, + "omsagent": { + "enabled": "[and(parameters('omsAgentEnabled'), not(empty(parameters('monitoringWorkspaceId'))))]", + "config": "[if(and(parameters('omsAgentEnabled'), not(empty(parameters('monitoringWorkspaceId')))), createObject('logAnalyticsWorkspaceResourceID', parameters('monitoringWorkspaceId')), null())]" + }, + "aciConnectorLinux": { + "enabled": "[parameters('aciConnectorLinuxEnabled')]" + }, + "azurepolicy": { + "enabled": "[parameters('azurePolicyEnabled')]", + "config": "[if(parameters('azurePolicyEnabled'), createObject('version', parameters('azurePolicyVersion')), null())]" + }, + "openServiceMesh": { + "enabled": "[parameters('openServiceMeshEnabled')]", + "config": "[if(parameters('openServiceMeshEnabled'), createObject(), null())]" + }, + "kubeDashboard": { + "enabled": "[parameters('kubeDashboardEnabled')]" + }, + "azureKeyvaultSecretsProvider": { + "enabled": "[parameters('enableKeyvaultSecretsProvider')]", + "config": "[if(parameters('enableKeyvaultSecretsProvider'), createObject('enableSecretRotation', toLower(string(parameters('enableSecretRotation')))), null())]" + } + }, + "oidcIssuerProfile": "[if(parameters('enableOidcIssuerProfile'), createObject('enabled', parameters('enableOidcIssuerProfile')), null())]", + "enableRBAC": "[parameters('enableRBAC')]", + "disableLocalAccounts": "[parameters('disableLocalAccounts')]", + "nodeResourceGroup": "[parameters('nodeResourceGroup')]", + "enablePodSecurityPolicy": "[parameters('enablePodSecurityPolicy')]", + "workloadAutoScalerProfile": { + "keda": { + "enabled": "[parameters('kedaAddon')]" + }, + "verticalPodAutoscaler": { + "enabled": "[parameters('vpaAddon')]" + } + }, + "networkProfile": { + "networkDataplane": "[parameters('networkDataplane')]", + "networkPlugin": "[parameters('networkPlugin')]", + "networkPluginMode": "[parameters('networkPluginMode')]", + "networkPolicy": "[parameters('networkPolicy')]", + "podCidr": "[parameters('podCidr')]", + "serviceCidr": "[parameters('serviceCidr')]", + "dnsServiceIP": "[parameters('dnsServiceIP')]", + "outboundType": "[parameters('outboundType')]", + "loadBalancerSku": "[parameters('loadBalancerSku')]", + "loadBalancerProfile": { + "managedOutboundIPs": "[if(not(equals(parameters('managedOutboundIPCount'), 0)), createObject('count', parameters('managedOutboundIPCount')), null())]", + "effectiveOutboundIPs": [], + "backendPoolType": "[parameters('backendPoolType')]" + } + }, + "publicNetworkAccess": "[parameters('publicNetworkAccess')]", + "aadProfile": { + "clientAppID": "[parameters('aadProfileClientAppID')]", + "serverAppID": "[parameters('aadProfileServerAppID')]", + "serverAppSecret": "[parameters('aadProfileServerAppSecret')]", + "managed": "[parameters('aadProfileManaged')]", + "enableAzureRBAC": "[parameters('aadProfileEnableAzureRBAC')]", + "adminGroupObjectIDs": "[parameters('aadProfileAdminGroupObjectIDs')]", + "tenantID": "[parameters('aadProfileTenantId')]" + }, + "autoScalerProfile": { + "balance-similar-node-groups": "[toLower(string(parameters('autoScalerProfileBalanceSimilarNodeGroups')))]", + "expander": "[parameters('autoScalerProfileExpander')]", + "max-empty-bulk-delete": "[parameters('autoScalerProfileMaxEmptyBulkDelete')]", + "max-graceful-termination-sec": "[parameters('autoScalerProfileMaxGracefulTerminationSec')]", + "max-node-provision-time": "[parameters('autoScalerProfileMaxNodeProvisionTime')]", + "max-total-unready-percentage": "[parameters('autoScalerProfileMaxTotalUnreadyPercentage')]", + "new-pod-scale-up-delay": "[parameters('autoScalerProfileNewPodScaleUpDelay')]", + "ok-total-unready-count": "[parameters('autoScalerProfileOkTotalUnreadyCount')]", + "scale-down-delay-after-add": "[parameters('autoScalerProfileScaleDownDelayAfterAdd')]", + "scale-down-delay-after-delete": "[parameters('autoScalerProfileScaleDownDelayAfterDelete')]", + "scale-down-delay-after-failure": "[parameters('autoScalerProfileScaleDownDelayAfterFailure')]", + "scale-down-unneeded-time": "[parameters('autoScalerProfileScaleDownUnneededTime')]", + "scale-down-unready-time": "[parameters('autoScalerProfileScaleDownUnreadyTime')]", + "scale-down-utilization-threshold": "[parameters('autoScalerProfileUtilizationThreshold')]", + "scan-interval": "[parameters('autoScalerProfileScanInterval')]", + "skip-nodes-with-local-storage": "[toLower(string(parameters('autoScalerProfileSkipNodesWithLocalStorage')))]", + "skip-nodes-with-system-pods": "[toLower(string(parameters('autoScalerProfileSkipNodesWithSystemPods')))]" + }, + "autoUpgradeProfile": { + "upgradeChannel": "[parameters('autoUpgradeProfileUpgradeChannel')]" + }, + "apiServerAccessProfile": { + "authorizedIPRanges": "[parameters('authorizedIPRanges')]", + "disableRunCommand": "[parameters('disableRunCommand')]", + "enablePrivateCluster": "[parameters('enablePrivateCluster')]", + "enablePrivateClusterPublicFQDN": "[parameters('enablePrivateClusterPublicFQDN')]", + "privateDNSZone": "[parameters('privateDNSZone')]" + }, + "azureMonitorProfile": { + "containerInsights": "[if(parameters('enableContainerInsights'), createObject('enabled', parameters('enableContainerInsights'), 'logAnalyticsWorkspaceResourceId', if(not(empty(parameters('monitoringWorkspaceId'))), parameters('monitoringWorkspaceId'), null()), 'disableCustomMetrics', parameters('disableCustomMetrics'), 'disablePrometheusMetricsScraping', parameters('disablePrometheusMetricsScraping'), 'syslogPort', parameters('syslogPort')), null())]", + "metrics": "[if(parameters('enableAzureMonitorProfileMetrics'), createObject('enabled', parameters('enableAzureMonitorProfileMetrics'), 'kubeStateMetrics', createObject('metricLabelsAllowlist', parameters('metricLabelsAllowlist'), 'metricAnnotationsAllowList', parameters('metricAnnotationsAllowList'))), null())]" + }, + "podIdentityProfile": { + "allowNetworkPluginKubenet": "[parameters('podIdentityProfileAllowNetworkPluginKubenet')]", + "enabled": "[parameters('podIdentityProfileEnable')]", + "userAssignedIdentities": "[parameters('podIdentityProfileUserAssignedIdentities')]", + "userAssignedIdentityExceptions": "[parameters('podIdentityProfileUserAssignedIdentityExceptions')]" + }, + "securityProfile": { + "defender": "[if(parameters('enableAzureDefender'), createObject('securityMonitoring', createObject('enabled', parameters('enableAzureDefender')), 'logAnalyticsWorkspaceResourceId', parameters('monitoringWorkspaceId')), null())]", + "workloadIdentity": "[if(parameters('enableWorkloadIdentity'), createObject('enabled', parameters('enableWorkloadIdentity')), null())]", + "imageCleaner": "[if(parameters('enableImageCleaner'), createObject('enabled', parameters('enableImageCleaner'), 'intervalHours', parameters('imageCleanerIntervalHours')), null())]" + }, + "storageProfile": { + "blobCSIDriver": { + "enabled": "[parameters('enableStorageProfileBlobCSIDriver')]" + }, + "diskCSIDriver": { + "enabled": "[if(and(equals(parameters('costAnalysisEnabled'), true()), not(equals(parameters('skuTier'), 'free'))), true(), parameters('enableStorageProfileDiskCSIDriver'))]" + }, + "fileCSIDriver": { + "enabled": "[parameters('enableStorageProfileFileCSIDriver')]" + }, + "snapshotController": { + "enabled": "[parameters('enableStorageProfileSnapshotController')]" + } + }, + "supportPlan": "[parameters('supportPlan')]" + } + }, + "managedCluster_lock": { + "condition": "[and(not(empty(coalesce(parameters('lock'), createObject()))), not(equals(tryGet(parameters('lock'), 'kind'), 'None')))]", + "type": "Microsoft.Authorization/locks", + "apiVersion": "2020-05-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('name'))]", + "name": "[coalesce(tryGet(parameters('lock'), 'name'), format('lock-{0}', parameters('name')))]", + "properties": { + "level": "[coalesce(tryGet(parameters('lock'), 'kind'), '')]", + "notes": "[if(equals(tryGet(parameters('lock'), 'kind'), 'CanNotDelete'), 'Cannot delete resource or child resources.', 'Cannot delete or modify the resource or child resources.')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "managedCluster_diagnosticSettings": { + "copy": { + "name": "managedCluster_diagnosticSettings", + "count": "[length(coalesce(parameters('diagnosticSettings'), createArray()))]" + }, + "type": "Microsoft.Insights/diagnosticSettings", + "apiVersion": "2021-05-01-preview", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('name'))]", + "name": "[coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'name'), format('{0}-diagnosticSettings', parameters('name')))]", + "properties": { + "copy": [ + { + "name": "metrics", + "count": "[length(coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'metricCategories'), createArray(createObject('category', 'AllMetrics'))))]", + "input": { + "category": "[coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'metricCategories'), createArray(createObject('category', 'AllMetrics')))[copyIndex('metrics')].category]", + "enabled": "[coalesce(tryGet(coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'metricCategories'), createArray(createObject('category', 'AllMetrics')))[copyIndex('metrics')], 'enabled'), true())]", + "timeGrain": null + } + }, + { + "name": "logs", + "count": "[length(coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'logCategoriesAndGroups'), createArray(createObject('categoryGroup', 'allLogs'))))]", + "input": { + "categoryGroup": "[tryGet(coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'logCategoriesAndGroups'), createArray(createObject('categoryGroup', 'allLogs')))[copyIndex('logs')], 'categoryGroup')]", + "category": "[tryGet(coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'logCategoriesAndGroups'), createArray(createObject('categoryGroup', 'allLogs')))[copyIndex('logs')], 'category')]", + "enabled": "[coalesce(tryGet(coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'logCategoriesAndGroups'), createArray(createObject('categoryGroup', 'allLogs')))[copyIndex('logs')], 'enabled'), true())]" + } + } + ], + "storageAccountId": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'storageAccountResourceId')]", + "workspaceId": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'workspaceResourceId')]", + "eventHubAuthorizationRuleId": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'eventHubAuthorizationRuleResourceId')]", + "eventHubName": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'eventHubName')]", + "marketplacePartnerId": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'marketplacePartnerResourceId')]", + "logAnalyticsDestinationType": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'logAnalyticsDestinationType')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "managedCluster_roleAssignments": { + "copy": { + "name": "managedCluster_roleAssignments", + "count": "[length(coalesce(variables('formattedRoleAssignments'), createArray()))]" + }, + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('name'))]", + "name": "[coalesce(tryGet(coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()], 'name'), guid(resourceId('Microsoft.ContainerService/managedClusters', parameters('name')), coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()].principalId, coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()].roleDefinitionId))]", + "properties": { + "roleDefinitionId": "[coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()].roleDefinitionId]", + "principalId": "[coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()].principalId]", + "description": "[tryGet(coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()], 'description')]", + "principalType": "[tryGet(coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()], 'principalType')]", + "condition": "[tryGet(coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()], 'condition')]", + "conditionVersion": "[if(not(empty(tryGet(coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()], 'condition'))), coalesce(tryGet(coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()], 'conditionVersion'), '2.0'), null())]", + "delegatedManagedIdentityResourceId": "[tryGet(coalesce(variables('formattedRoleAssignments'), createArray())[copyIndex()], 'delegatedManagedIdentityResourceId')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "dnsZone": { + "condition": "[and(and(equals(parameters('enableDnsZoneContributorRoleAssignment'), true()), not(equals(parameters('dnsZoneResourceId'), null()))), parameters('webApplicationRoutingEnabled'))]", + "existing": true, + "type": "Microsoft.Network/dnsZones", + "apiVersion": "2018-05-01", + "name": "[last(split(if(not(empty(parameters('dnsZoneResourceId'))), parameters('dnsZoneResourceId'), '/dummmyZone'), '/'))]" + }, + "dnsZone_roleAssignment": { + "condition": "[and(and(equals(parameters('enableDnsZoneContributorRoleAssignment'), true()), not(equals(parameters('dnsZoneResourceId'), null()))), parameters('webApplicationRoutingEnabled'))]", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[format('Microsoft.Network/dnsZones/{0}', last(split(if(not(empty(parameters('dnsZoneResourceId'))), parameters('dnsZoneResourceId'), '/dummmyZone'), '/')))]", + "name": "[guid(resourceId('Microsoft.Network/dnsZones', last(split(if(not(empty(parameters('dnsZoneResourceId'))), parameters('dnsZoneResourceId'), '/dummmyZone'), '/'))), subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314'), 'DNS Zone Contributor')]", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314')]", + "principalId": "[reference('managedCluster').ingressProfile.webAppRouting.identity.objectId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "dnsZone", + "managedCluster" + ] + }, + "managedCluster_maintenanceConfigurations": { + "condition": "[not(empty(parameters('maintenanceConfiguration')))]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-ManagedCluster-MaintenanceConfigurations', uniqueString(deployment().name, parameters('location')))]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "maintenanceWindow": { + "value": "[parameters('maintenanceConfiguration').maintenanceWindow]" + }, + "managedClusterName": { + "value": "[parameters('name')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.29.47.4906", + "templateHash": "12168542117744033419" + }, + "name": "Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations", + "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "maintenanceWindow": { + "type": "object", + "metadata": { + "description": "Required. Maintenance window for the maintenance configuration." + } + }, + "managedClusterName": { + "type": "string", + "metadata": { + "description": "Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment." + } + }, + "name": { + "type": "string", + "defaultValue": "aksManagedAutoUpgradeSchedule", + "metadata": { + "description": "Optional. Name of the maintenance configuration." + } + } + }, + "resources": [ + { + "type": "Microsoft.ContainerService/managedClusters/maintenanceConfigurations", + "apiVersion": "2023-10-01", + "name": "[format('{0}/{1}', parameters('managedClusterName'), parameters('name'))]", + "properties": { + "maintenanceWindow": "[parameters('maintenanceWindow')]" + } + } + ], + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the maintenance configuration." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the maintenance configuration." + }, + "value": "[resourceId('Microsoft.ContainerService/managedClusters/maintenanceConfigurations', parameters('managedClusterName'), parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The resource group the agent pool was deployed into." + }, + "value": "[resourceGroup().name]" + } + } + } + }, + "dependsOn": [ + "managedCluster" + ] + }, + "managedCluster_agentPools": { + "copy": { + "name": "managedCluster_agentPools", + "count": "[length(coalesce(parameters('agentPools'), createArray()))]" + }, + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-ManagedCluster-AgentPool-{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "managedClusterName": { + "value": "[parameters('name')]" + }, + "name": { + "value": "[coalesce(parameters('agentPools'), createArray())[copyIndex()].name]" + }, + "availabilityZones": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'availabilityZones')]" + }, + "count": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'count')]" + }, + "sourceResourceId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'sourceResourceId')]" + }, + "enableAutoScaling": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableAutoScaling')]" + }, + "enableEncryptionAtHost": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableEncryptionAtHost')]" + }, + "enableFIPS": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableFIPS')]" + }, + "enableNodePublicIP": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableNodePublicIP')]" + }, + "enableUltraSSD": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableUltraSSD')]" + }, + "gpuInstanceProfile": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'gpuInstanceProfile')]" + }, + "kubeletDiskType": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'kubeletDiskType')]" + }, + "maxCount": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'maxCount')]" + }, + "maxPods": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'maxPods')]" + }, + "minCount": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'minCount')]" + }, + "mode": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'mode')]" + }, + "nodeLabels": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodeLabels')]" + }, + "nodePublicIpPrefixId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodePublicIpPrefixId')]" + }, + "nodeTaints": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodeTaints')]" + }, + "orchestratorVersion": { + "value": "[coalesce(tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'orchestratorVersion'), parameters('kubernetesVersion'))]" + }, + "osDiskSizeGB": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osDiskSizeGB')]" + }, + "osDiskType": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osDiskType')]" + }, + "osSku": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osSku')]" + }, + "osType": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osType')]" + }, + "podSubnetId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'podSubnetId')]" + }, + "proximityPlacementGroupResourceId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'proximityPlacementGroupResourceId')]" + }, + "scaleDownMode": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'scaleDownMode')]" + }, + "scaleSetEvictionPolicy": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'scaleSetEvictionPolicy')]" + }, + "scaleSetPriority": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'scaleSetPriority')]" + }, + "spotMaxPrice": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'spotMaxPrice')]" + }, + "tags": { + "value": "[coalesce(tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'tags'), parameters('tags'))]" + }, + "type": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'type')]" + }, + "maxSurge": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'maxSurge')]" + }, + "vmSize": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'vmSize')]" + }, + "vnetSubnetId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'vnetSubnetId')]" + }, + "workloadRuntime": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'workloadRuntime')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.29.47.4906", + "templateHash": "2004205618690542488" + }, + "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", + "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "managedClusterName": { + "type": "string", + "metadata": { + "description": "Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment." + } + }, + "name": { + "type": "string", + "metadata": { + "description": "Required. Name of the agent pool." + } + }, + "availabilityZones": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is \"VirtualMachineScaleSets\"." + } + }, + "count": { + "type": "int", + "defaultValue": 1, + "minValue": 0, + "maxValue": 1000, + "metadata": { + "description": "Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1." + } + }, + "sourceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. This is the ARM ID of the source object to be used to create the target object." + } + }, + "enableAutoScaling": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable auto-scaler." + } + }, + "enableEncryptionAtHost": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled." + } + }, + "enableFIPS": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details." + } + }, + "enableNodePublicIP": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools)." + } + }, + "enableUltraSSD": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable UltraSSD." + } + }, + "gpuInstanceProfile": { + "type": "string", + "nullable": true, + "allowedValues": [ + "MIG1g", + "MIG2g", + "MIG3g", + "MIG4g", + "MIG7g" + ], + "metadata": { + "description": "Optional. GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU." + } + }, + "kubeletDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage." + } + }, + "maxCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of nodes for auto-scaling." + } + }, + "maxPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of pods that can run on a node." + } + }, + "minCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of nodes for auto-scaling." + } + }, + "mode": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. A cluster must have at least one \"System\" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools." + } + }, + "nodeLabels": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The node labels to be persisted across all nodes in agent pool." + } + }, + "nodePublicIpPrefixId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. ResourceId of the node PublicIPPrefix." + } + }, + "nodeTaints": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule." + } + }, + "orchestratorVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool)." + } + }, + "osDiskSizeGB": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified." + } + }, + "osDiskType": { + "type": "string", + "nullable": true, + "allowedValues": [ + "Ephemeral", + "Managed" + ], + "metadata": { + "description": "Optional. The default is \"Ephemeral\" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to \"Managed\". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os)." + } + }, + "osSku": { + "type": "string", + "nullable": true, + "allowedValues": [ + "AzureLinux", + "CBLMariner", + "Ubuntu", + "Windows2019", + "Windows2022" + ], + "metadata": { + "description": "Optional. Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows." + } + }, + "osType": { + "type": "string", + "defaultValue": "Linux", + "allowedValues": [ + "Linux", + "Windows" + ], + "metadata": { + "description": "Optional. The operating system type. The default is Linux." + } + }, + "podSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + } + }, + "proximityPlacementGroupResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The ID for the Proximity Placement Group." + } + }, + "scaleDownMode": { + "type": "string", + "defaultValue": "Delete", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "metadata": { + "description": "Optional. Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing)." + } + }, + "scaleSetEvictionPolicy": { + "type": "string", + "defaultValue": "Delete", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "metadata": { + "description": "Optional. The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs." + } + }, + "scaleSetPriority": { + "type": "string", + "nullable": true, + "allowedValues": [ + "Regular", + "Spot" + ], + "metadata": { + "description": "Optional. The Virtual Machine Scale Set priority." + } + }, + "spotMaxPrice": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing)." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Tags of the resource." + } + }, + "type": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The type of Agent Pool." + } + }, + "maxSurge": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. This can either be set to an integer (e.g. \"5\") or a percentage (e.g. \"50%\"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade." + } + }, + "vmSize": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions." + } + }, + "vnetSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + } + }, + "workloadRuntime": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Determines the type of workload a node can run." + } + } + }, + "resources": { + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2024-03-02-preview", + "name": "[parameters('managedClusterName')]" + }, + "agentPool": { + "type": "Microsoft.ContainerService/managedClusters/agentPools", + "apiVersion": "2023-07-02-preview", + "name": "[format('{0}/{1}', parameters('managedClusterName'), parameters('name'))]", + "properties": { + "availabilityZones": "[parameters('availabilityZones')]", + "count": "[parameters('count')]", + "creationData": "[if(not(empty(parameters('sourceResourceId'))), createObject('sourceResourceId', parameters('sourceResourceId')), null())]", + "enableAutoScaling": "[parameters('enableAutoScaling')]", + "enableEncryptionAtHost": "[parameters('enableEncryptionAtHost')]", + "enableFIPS": "[parameters('enableFIPS')]", + "enableNodePublicIP": "[parameters('enableNodePublicIP')]", + "enableUltraSSD": "[parameters('enableUltraSSD')]", + "gpuInstanceProfile": "[parameters('gpuInstanceProfile')]", + "kubeletDiskType": "[parameters('kubeletDiskType')]", + "maxCount": "[parameters('maxCount')]", + "maxPods": "[parameters('maxPods')]", + "minCount": "[parameters('minCount')]", + "mode": "[parameters('mode')]", + "nodeLabels": "[parameters('nodeLabels')]", + "nodePublicIPPrefixID": "[parameters('nodePublicIpPrefixId')]", + "nodeTaints": "[parameters('nodeTaints')]", + "orchestratorVersion": "[parameters('orchestratorVersion')]", + "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "osDiskType": "[parameters('osDiskType')]", + "osSKU": "[parameters('osSku')]", + "osType": "[parameters('osType')]", + "podSubnetID": "[parameters('podSubnetId')]", + "proximityPlacementGroupID": "[parameters('proximityPlacementGroupResourceId')]", + "scaleDownMode": "[parameters('scaleDownMode')]", + "scaleSetEvictionPolicy": "[parameters('scaleSetEvictionPolicy')]", + "scaleSetPriority": "[parameters('scaleSetPriority')]", + "spotMaxPrice": "[parameters('spotMaxPrice')]", + "tags": "[parameters('tags')]", + "type": "[parameters('type')]", + "upgradeSettings": { + "maxSurge": "[parameters('maxSurge')]" + }, + "vmSize": "[parameters('vmSize')]", + "vnetSubnetID": "[parameters('vnetSubnetId')]", + "workloadRuntime": "[parameters('workloadRuntime')]" + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the agent pool." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the agent pool." + }, + "value": "[resourceId('Microsoft.ContainerService/managedClusters/agentPools', parameters('managedClusterName'), parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The resource group the agent pool was deployed into." + }, + "value": "[resourceGroup().name]" + } + } + } + }, + "dependsOn": [ + "managedCluster" + ] + }, + "managedCluster_extension": { + "condition": "[not(empty(parameters('fluxExtension')))]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-ManagedCluster-FluxExtension', uniqueString(deployment().name, parameters('location')))]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "clusterName": { + "value": "[parameters('name')]" + }, + "configurationProtectedSettings": { + "value": "[tryGet(parameters('fluxExtension'), 'configurationProtectedSettings')]" + }, + "configurationSettings": { + "value": "[tryGet(parameters('fluxExtension'), 'configurationSettings')]" + }, + "enableTelemetry": { + "value": "[parameters('enableTelemetry')]" + }, + "extensionType": { + "value": "microsoft.flux" + }, + "fluxConfigurations": { + "value": "[tryGet(parameters('fluxExtension'), 'configurations')]" + }, + "location": { + "value": "[parameters('location')]" + }, + "name": { + "value": "flux" + }, + "releaseNamespace": { + "value": "[coalesce(tryGet(parameters('fluxExtension'), 'releaseNamespace'), 'flux-system')]" + }, + "releaseTrain": { + "value": "[coalesce(tryGet(parameters('fluxExtension'), 'releaseTrain'), 'Stable')]" + }, + "version": { + "value": "[tryGet(parameters('fluxExtension'), 'version')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.23.1.45101", + "templateHash": "12293754418506359991" + }, + "name": "Kubernetes Configuration Extensions", + "description": "This module deploys a Kubernetes Configuration Extension.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. The name of the Flux Configuration." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "clusterName": { + "type": "string", + "metadata": { + "description": "Required. The name of the AKS cluster that should be configured." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Location for all resources." + } + }, + "configurationProtectedSettings": { + "type": "secureObject", + "nullable": true, + "metadata": { + "description": "Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension." + } + }, + "configurationSettings": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Configuration settings, as name-value pairs for configuring this extension." + } + }, + "extensionType": { + "type": "string", + "metadata": { + "description": "Required. Type of the extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the extension publisher." + } + }, + "releaseTrain": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is \"true\"." + } + }, + "releaseNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created." + } + }, + "targetNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created." + } + }, + "version": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Version of the extension for this extension, if it is \"pinned\" to a specific version." + } + }, + "fluxConfigurations": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. A list of flux configuraitons." + } + } + }, + "resources": { + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2023-07-01", + "name": "[format('46d3xbcp.res.kubernetesconfiguration-fluxconfig.{0}.{1}', replace('0.2.0', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2022-07-01", + "name": "[parameters('clusterName')]" + }, + "extension": { + "type": "Microsoft.KubernetesConfiguration/extensions", + "apiVersion": "2022-03-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('clusterName'))]", + "name": "[parameters('name')]", + "properties": { + "autoUpgradeMinorVersion": "[if(not(empty(parameters('version'))), false(), true())]", + "configurationProtectedSettings": "[parameters('configurationProtectedSettings')]", + "configurationSettings": "[parameters('configurationSettings')]", + "extensionType": "[parameters('extensionType')]", + "releaseTrain": "[parameters('releaseTrain')]", + "scope": { + "cluster": "[if(not(empty(coalesce(parameters('releaseNamespace'), ''))), createObject('releaseNamespace', parameters('releaseNamespace')), null())]", + "namespace": "[if(not(empty(coalesce(parameters('targetNamespace'), ''))), createObject('targetNamespace', parameters('targetNamespace')), null())]" + }, + "version": "[parameters('version')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "fluxConfiguration": { + "copy": { + "name": "fluxConfiguration", + "count": "[length(coalesce(parameters('fluxConfigurations'), createArray()))]" + }, + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-ManagedCluster-FluxConfiguration{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "enableTelemetry": { + "value": "[parameters('enableTelemetry')]" + }, + "clusterName": { + "value": "[parameters('clusterName')]" + }, + "scope": { + "value": "[coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()].scope]" + }, + "namespace": { + "value": "[coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()].namespace]" + }, + "sourceKind": "[if(contains(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'gitRepository'), createObject('value', 'GitRepository'), createObject('value', 'Bucket'))]", + "name": { + "value": "[coalesce(tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'name'), toLower(format('{0}-fluxconfiguration{1}', parameters('clusterName'), copyIndex())))]" + }, + "bucket": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'bucket')]" + }, + "configurationProtectedSettings": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'configurationProtectedSettings')]" + }, + "gitRepository": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'gitRepository')]" + }, + "kustomizations": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'kustomizations')]" + }, + "suspend": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'suspend')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.23.1.45101", + "templateHash": "13420454476526931427" + }, + "name": "Kubernetes Configuration Flux Configurations", + "description": "This module deploys a Kubernetes Configuration Flux Configuration.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. The name of the Flux Configuration." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "clusterName": { + "type": "string", + "metadata": { + "description": "Required. The name of the AKS cluster that should be configured." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Location for all resources." + } + }, + "bucket": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `Bucket`." + } + }, + "configurationProtectedSettings": { + "type": "secureObject", + "nullable": true, + "metadata": { + "description": "Optional. Key-value pairs of protected configuration settings for the configuration." + } + }, + "gitRepository": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `GitRepository`." + } + }, + "kustomizations": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster." + } + }, + "namespace": { + "type": "string", + "metadata": { + "description": "Required. The namespace to which this configuration is installed to. Maximum of 253 lower case alphanumeric characters, hyphen and period only." + } + }, + "scope": { + "type": "string", + "allowedValues": [ + "cluster", + "namespace" + ], + "metadata": { + "description": "Required. Scope at which the configuration will be installed." + } + }, + "sourceKind": { + "type": "string", + "allowedValues": [ + "Bucket", + "GitRepository" + ], + "metadata": { + "description": "Required. Source Kind to pull the configuration data from." + } + }, + "suspend": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether this configuration should suspend its reconciliation of its kustomizations and sources." + } + } + }, + "resources": { + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2023-07-01", + "name": "[format('46d3xbcp.res.kubernetesconfiguration-extension.{0}.{1}', replace('0.2.0', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2022-07-01", + "name": "[parameters('clusterName')]" + }, + "fluxConfiguration": { + "type": "Microsoft.KubernetesConfiguration/fluxConfigurations", + "apiVersion": "2022-03-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('clusterName'))]", + "name": "[parameters('name')]", + "properties": { + "bucket": "[parameters('bucket')]", + "configurationProtectedSettings": "[parameters('configurationProtectedSettings')]", + "gitRepository": "[parameters('gitRepository')]", + "kustomizations": "[parameters('kustomizations')]", + "namespace": "[parameters('namespace')]", + "scope": "[parameters('scope')]", + "sourceKind": "[parameters('sourceKind')]", + "suspend": "[parameters('suspend')]" + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the flux configuration." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the flux configuration." + }, + "value": "[extensionResourceId(resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), 'Microsoft.KubernetesConfiguration/fluxConfigurations', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The name of the resource group the flux configuration was deployed into." + }, + "value": "[resourceGroup().name]" + } + } + } + }, + "dependsOn": [ + "extension", + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the extension." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the extension." + }, + "value": "[extensionResourceId(resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), 'Microsoft.KubernetesConfiguration/extensions', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The name of the resource group the extension was deployed into." + }, + "value": "[resourceGroup().name]" + } + } + } + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the managed cluster." + }, + "value": "[resourceId('Microsoft.ContainerService/managedClusters', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The resource group the managed cluster was deployed into." + }, + "value": "[resourceGroup().name]" + }, + "name": { + "type": "string", + "metadata": { + "description": "The name of the managed cluster." + }, + "value": "[parameters('name')]" + }, + "controlPlaneFQDN": { + "type": "string", + "metadata": { + "description": "The control plane FQDN of the managed cluster." + }, + "value": "[if(parameters('enablePrivateCluster'), reference('managedCluster').privateFQDN, reference('managedCluster').fqdn)]" + }, + "systemAssignedMIPrincipalId": { + "type": "string", + "metadata": { + "description": "The principal ID of the system assigned identity." + }, + "value": "[coalesce(tryGet(tryGet(reference('managedCluster', '2024-03-02-preview', 'full'), 'identity'), 'principalId'), '')]" + }, + "kubeletIdentityClientId": { + "type": "string", + "metadata": { + "description": "The Client ID of the AKS identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(reference('managedCluster'), 'identityProfile'), 'kubeletidentity'), 'clientId'), '')]" + }, + "kubeletIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of the AKS identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(reference('managedCluster'), 'identityProfile'), 'kubeletidentity'), 'objectId'), '')]" + }, + "kubeletIdentityResourceId": { + "type": "string", + "metadata": { + "description": "The Resource ID of the AKS identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(reference('managedCluster'), 'identityProfile'), 'kubeletidentity'), 'resourceId'), '')]" + }, + "omsagentIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of the OMS agent identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'addonProfiles'), 'omsagent'), 'identity'), 'objectId'), '')]" + }, + "keyvaultIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of the Key Vault Secrets Provider identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'addonProfiles'), 'azureKeyvaultSecretsProvider'), 'identity'), 'objectId'), '')]" + }, + "keyvaultIdentityClientId": { + "type": "string", + "metadata": { + "description": "The Client ID of the Key Vault Secrets Provider identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'addonProfiles'), 'azureKeyvaultSecretsProvider'), 'identity'), 'clientId'), '')]" + }, + "ingressApplicationGatewayIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of Application Gateway Ingress Controller (AGIC) identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'addonProfiles'), 'ingressApplicationGateway'), 'identity'), 'objectId'), '')]" + }, + "location": { + "type": "string", + "metadata": { + "description": "The location the resource was deployed into." + }, + "value": "[reference('managedCluster', '2024-03-02-preview', 'full').location]" + }, + "oidcIssuerUrl": { + "type": "string", + "metadata": { + "description": "The OIDC token issuer URL." + }, + "value": "[coalesce(tryGet(tryGet(reference('managedCluster'), 'oidcIssuerProfile'), 'issuerURL'), '')]" + }, + "addonProfiles": { + "type": "object", + "metadata": { + "description": "The addonProfiles of the Kubernetes cluster." + }, + "value": "[coalesce(tryGet(reference('managedCluster'), 'addonProfiles'), createObject())]" + }, + "webAppRoutingIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of Web Application Routing." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'ingressProfile'), 'webAppRouting'), 'identity'), 'objectId'), '')]" + } + } +} \ No newline at end of file diff --git a/bicep/modules/managed-cluster/maintenance-configurations/README.md b/bicep/modules/managed-cluster/maintenance-configurations/README.md new file mode 100644 index 00000000..ddaaa88a --- /dev/null +++ b/bicep/modules/managed-cluster/maintenance-configurations/README.md @@ -0,0 +1,65 @@ +# Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations `[Microsoft.ContainerService/managedClusters/maintenanceConfigurations]` + +This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations. + +## Navigation + +- [Resource Types](#Resource-Types) +- [Parameters](#Parameters) +- [Outputs](#Outputs) + +## Resource Types + +| Resource Type | API Version | +| :-- | :-- | +| `Microsoft.ContainerService/managedClusters/maintenanceConfigurations` | [2023-10-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2023-10-01/managedClusters/maintenanceConfigurations) | + +## Parameters + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`maintenanceWindow`](#parameter-maintenancewindow) | object | Maintenance window for the maintenance configuration. | + +**Conditional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`managedClusterName`](#parameter-managedclustername) | string | The name of the parent managed cluster. Required if the template is used in a standalone deployment. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-name) | string | Name of the maintenance configuration. | + +### Parameter: `maintenanceWindow` + +Maintenance window for the maintenance configuration. + +- Required: Yes +- Type: object + +### Parameter: `managedClusterName` + +The name of the parent managed cluster. Required if the template is used in a standalone deployment. + +- Required: Yes +- Type: string + +### Parameter: `name` + +Name of the maintenance configuration. + +- Required: No +- Type: string +- Default: `'aksManagedAutoUpgradeSchedule'` + +## Outputs + +| Output | Type | Description | +| :-- | :-- | :-- | +| `name` | string | The name of the maintenance configuration. | +| `resourceGroupName` | string | The resource group the agent pool was deployed into. | +| `resourceId` | string | The resource ID of the maintenance configuration. | diff --git a/bicep/modules/managed-cluster/maintenance-configurations/main.bicep b/bicep/modules/managed-cluster/maintenance-configurations/main.bicep new file mode 100644 index 00000000..52f609f0 --- /dev/null +++ b/bicep/modules/managed-cluster/maintenance-configurations/main.bicep @@ -0,0 +1,33 @@ +metadata name = 'Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations' +metadata description = 'This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations.' +metadata owner = 'Azure/module-maintainers' + +@description('Required. Maintenance window for the maintenance configuration.') +param maintenanceWindow object + +@description('Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment.') +param managedClusterName string + +@description('Optional. Name of the maintenance configuration.') +param name string = 'aksManagedAutoUpgradeSchedule' + +resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-preview' existing = { + name: managedClusterName +} + +resource aksManagedAutoUpgradeSchedule 'Microsoft.ContainerService/managedClusters/maintenanceConfigurations@2023-10-01' = { + name: name + parent: managedCluster + properties: { + maintenanceWindow: maintenanceWindow + } +} + +@description('The name of the maintenance configuration.') +output name string = aksManagedAutoUpgradeSchedule.name + +@description('The resource ID of the maintenance configuration.') +output resourceId string = aksManagedAutoUpgradeSchedule.id + +@description('The resource group the agent pool was deployed into.') +output resourceGroupName string = resourceGroup().name diff --git a/bicep/modules/managed-cluster/maintenance-configurations/main.json b/bicep/modules/managed-cluster/maintenance-configurations/main.json new file mode 100644 index 00000000..3c4f84d1 --- /dev/null +++ b/bicep/modules/managed-cluster/maintenance-configurations/main.json @@ -0,0 +1,68 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.29.47.4906", + "templateHash": "12168542117744033419" + }, + "name": "Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations", + "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "maintenanceWindow": { + "type": "object", + "metadata": { + "description": "Required. Maintenance window for the maintenance configuration." + } + }, + "managedClusterName": { + "type": "string", + "metadata": { + "description": "Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment." + } + }, + "name": { + "type": "string", + "defaultValue": "aksManagedAutoUpgradeSchedule", + "metadata": { + "description": "Optional. Name of the maintenance configuration." + } + } + }, + "resources": [ + { + "type": "Microsoft.ContainerService/managedClusters/maintenanceConfigurations", + "apiVersion": "2023-10-01", + "name": "[format('{0}/{1}', parameters('managedClusterName'), parameters('name'))]", + "properties": { + "maintenanceWindow": "[parameters('maintenanceWindow')]" + } + } + ], + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the maintenance configuration." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the maintenance configuration." + }, + "value": "[resourceId('Microsoft.ContainerService/managedClusters/maintenanceConfigurations', parameters('managedClusterName'), parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The resource group the agent pool was deployed into." + }, + "value": "[resourceGroup().name]" + } + } +} \ No newline at end of file diff --git a/bicep/modules/managed-cluster/tests/e2e/automatic/main.test.bicep b/bicep/modules/managed-cluster/tests/e2e/automatic/main.test.bicep new file mode 100644 index 00000000..5d8d63a8 --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/automatic/main.test.bicep @@ -0,0 +1,73 @@ +targetScope = 'subscription' + +metadata name = 'Using only defaults and use AKS Automatic mode' +metadata description = 'This instance deploys the module with the set of automatic parameters.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csauto' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: resourceLocation +} + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + maintenanceConfiguration: { + maintenanceWindow: { + schedule: { + daily: null + weekly: { + intervalWeeks: 1 + dayOfWeek: 'Sunday' + } + absoluteMonthly: null + relativeMonthly: null + } + durationHours: 4 + utcOffset: '+00:00' + startDate: '2024-07-03' + startTime: '00:00' + } + } + managedIdentities: { + systemAssigned: true + } + + primaryAgentPoolProfile: [ + { + name: 'systempool' + count: 3 + vmSize: 'Standard_DS2_v2' + mode: 'System' + } + ] + } + } +] diff --git a/bicep/modules/managed-cluster/tests/e2e/azure/dependencies.bicep b/bicep/modules/managed-cluster/tests/e2e/azure/dependencies.bicep new file mode 100644 index 00000000..3819c5f5 --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/azure/dependencies.bicep @@ -0,0 +1,187 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Virtual Network to create.') +param virtualNetworkName string + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The name of the Kubelet Identity Managed Identity to create.') +param managedIdentityKubeletIdentityName string + +@description('Required. The name of the Disk Encryption Set to create.') +param diskEncryptionSetName string + +@description('Required. The name of the Key Vault to create.') +param keyVaultName string + +@description('Required. The name of the Proximity Placement Group to create.') +param proximityPlacementGroupName string + +@description('Required. The name of the DNS Zone to create.') +param dnsZoneName string + +@description('Required. The name of the log analytics workspace to create.') +param logAnalyticsWorkspaceName string + +var addressPrefix = '10.1.0.0/22' + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefix + ] + } + subnets: map(range(0, 3), i => { + name: 'subnet-${i}' + properties: { + addressPrefix: cidrSubnet(addressPrefix, 24, i) + } + }) + } +} + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: managedIdentityName + location: location +} + +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { + name: logAnalyticsWorkspaceName + location: location +} + +resource keyVault 'Microsoft.KeyVault/vaults@2022-11-01' = { + name: keyVaultName + location: location + properties: { + sku: { + family: 'A' + name: 'standard' + } + tenantId: tenant().tenantId + enablePurgeProtection: true // Required for encryption to work + softDeleteRetentionInDays: 7 + enabledForTemplateDeployment: true + enabledForDiskEncryption: true + enabledForDeployment: true + enableRbacAuthorization: true + accessPolicies: [] + } + + resource key 'keys@2022-07-01' = { + name: 'encryptionKey' + properties: { + kty: 'RSA' + } + } + + resource kmskey 'keys@2022-07-01' = { + name: 'kmsEncryptionKey' + properties: { + kty: 'RSA' + } + } +} + +resource keyPermissionsKeyVaultCryptoUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Crypto-User-RoleAssignment') + scope: keyVault + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '12338af0-0e69-4776-bea7-57ae8d297424' + ) // KeyVault-Crypto-User + principalType: 'ServicePrincipal' + } +} + +resource diskEncryptionSet 'Microsoft.Compute/diskEncryptionSets@2022-07-02' = { + name: diskEncryptionSetName + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentity.id}': {} + } + } + properties: { + activeKey: { + sourceVault: { + id: keyVault.id + } + keyUrl: keyVault::key.properties.keyUriWithVersion + } + encryptionType: 'EncryptionAtRestWithCustomerKey' + } + dependsOn: [ + keyPermissionsKeyVaultCryptoUser + ] +} + +resource proximityPlacementGroup 'Microsoft.Compute/proximityPlacementGroups@2022-03-01' = { + name: proximityPlacementGroupName + location: location +} + +@description('The resource ID of the created Virtual Network Subnet.') +output subnetResourceIds array = [ + virtualNetwork.properties.subnets[0].id + virtualNetwork.properties.subnets[1].id + virtualNetwork.properties.subnets[2].id +] + +resource dnsZone 'Microsoft.Network/dnsZones@2018-05-01' = { + name: dnsZoneName + location: 'global' +} + +resource managedIdentityKubeletIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: managedIdentityKubeletIdentityName + location: location +} + +resource roleAssignmentKubeletIdentity 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${location}-${managedIdentityKubeletIdentity.id}-ManagedIdentityOperator-RoleAssignment') + scope: managedIdentityKubeletIdentity + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'f1a07417-d97a-45cb-824c-7a7467783830' + ) // Managed Identity Operator Role used for Kubelet identity. + principalType: 'ServicePrincipal' + } +} + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The resource ID of the created Kubelet Identity Managed Identity.') +output managedIdentityKubeletIdentityResourceId string = managedIdentityKubeletIdentity.id + +@description('The resource ID of the created Disk Encryption Set.') +output diskEncryptionSetResourceId string = diskEncryptionSet.id + +@description('The resource ID of the created Key Vault.') +output keyVaultResourceId string = keyVault.id + +@description('The name of the Key Vault Encryption Key.') +output keyVaultEncryptionKeyName string = keyVault::key.name + +@description('The resource ID of the created Proximity Placement Group.') +output proximityPlacementGroupResourceId string = proximityPlacementGroup.id + +@description('The resource ID of the created DNS Zone.') +output dnsZoneResourceId string = dnsZone.id + +@description('The resource ID of the created Log Analytics Workspace.') +output logAnalyticsWorkspaceResourceId string = logAnalyticsWorkspace.id diff --git a/bicep/modules/managed-cluster/tests/e2e/azure/main.test.bicep b/bicep/modules/managed-cluster/tests/e2e/azure/main.test.bicep new file mode 100644 index 00000000..9032ea28 --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/azure/main.test.bicep @@ -0,0 +1,288 @@ +targetScope = 'subscription' +metadata name = 'Using Azure CNI Network Plugin.' +metadata description = 'This instance deploys the module with Azure CNI network plugin .' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmaz' + +@description('Generated. Used as a basis for unique resource names.') +param baseTime string = utcNow('u') + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}' + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + managedIdentityKubeletIdentityName: 'dep-${namePrefix}-msiki-${serviceShort}' + diskEncryptionSetName: 'dep-${namePrefix}-des-${serviceShort}' + proximityPlacementGroupName: 'dep-${namePrefix}-ppg-${serviceShort}' + // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total) + keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}' + dnsZoneName: 'dep-${namePrefix}-dns-${serviceShort}.com' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + location: resourceLocation + } +} + +// Diagnostics +// =========== +module diagnosticDependencies '../../../../../../utilities/e2e-template-assets/templates/diagnostic.dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-diagnosticDependencies' + params: { + storageAccountName: 'dep${namePrefix}diasa${serviceShort}01' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}' + eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + location: resourceLocation + name: '${namePrefix}${serviceShort}001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[0] + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[1] + proximityPlacementGroupResourceId: nestedDependencies.outputs.proximityPlacementGroupResourceId + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[2] + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + enableWorkloadIdentity: true + enableOidcIssuerProfile: true + networkPlugin: 'azure' + networkDataplane: 'azure' + networkPluginMode: 'overlay' + diagnosticSettings: [ + { + name: 'customSetting' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + eventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + eventHubAuthorizationRuleResourceId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + storageAccountResourceId: diagnosticDependencies.outputs.storageAccountResourceId + workspaceResourceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + } + ] + diskEncryptionSetResourceId: nestedDependencies.outputs.diskEncryptionSetResourceId + openServiceMeshEnabled: true + enableStorageProfileBlobCSIDriver: true + enableStorageProfileDiskCSIDriver: true + enableStorageProfileFileCSIDriver: true + enableStorageProfileSnapshotController: true + managedIdentities: { + userAssignedResourcesIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] + } + identityProfile: { + kubeletidentity: { + resourceId: nestedDependencies.outputs.managedIdentityKubeletIdentityResourceId + } + } + omsAgentEnabled: true + monitoringWorkspaceId: nestedDependencies.outputs.logAnalyticsWorkspaceResourceId + enableAzureDefender: true + enableKeyvaultSecretsProvider: true + enablePodSecurityPolicy: false + enableAzureMonitorProfileMetrics: true + customerManagedKey: { + keyName: nestedDependencies.outputs.keyVaultEncryptionKeyName + keyVaultNetworkAccess: 'Public' + keyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId + } + lock: { + kind: 'CanNotDelete' + name: 'myCustomLockName' + } + roleAssignments: [ + { + name: 'ac915208-669e-4665-9792-7e2dc861f569' + roleDefinitionIdOrName: 'Owner' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + { + name: guid('Custom seed ${namePrefix}${serviceShort}') + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + { + roleDefinitionIdOrName: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'acdd72a7-3385-48ef-bd42-f606fba81ae7' + ) + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + ] + tags: { + 'hidden-title': 'This is visible in the resource name' + Environment: 'Non-Prod' + Role: 'DeploymentValidation' + } + fluxExtension: { + configurationSettings: { + 'helm-controller.enabled': 'true' + 'source-controller.enabled': 'true' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + } + configurations: [ + { + namespace: 'flux-system' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + } + { + namespace: 'flux-system-helm' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + infra: { + path: './infrastructure' + dependsOn: [] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + validation: 'none' + prune: true + } + apps: { + path: './apps/staging' + dependsOn: [ + 'infra' + ] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + retryIntervalInSeconds: 120 + prune: true + } + } + } + ] + } + } + dependsOn: [ + nestedDependencies + diagnosticDependencies + ] + } +] diff --git a/bicep/modules/managed-cluster/tests/e2e/defaults/main.test.bicep b/bicep/modules/managed-cluster/tests/e2e/defaults/main.test.bicep new file mode 100644 index 00000000..60a4103d --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/defaults/main.test.bicep @@ -0,0 +1,55 @@ +targetScope = 'subscription' + +metadata name = 'Using only defaults' +metadata description = 'This instance deploys the module with the minimum set of required parameters.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmin' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: resourceLocation +} + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + managedIdentities: { + systemAssigned: true + } + primaryAgentPoolProfile: [ + { + name: 'systempool' + count: 3 + vmSize: 'Standard_DS2_v2' + mode: 'System' + } + ] + } + } +] diff --git a/bicep/modules/managed-cluster/tests/e2e/kubenet/dependencies.bicep b/bicep/modules/managed-cluster/tests/e2e/kubenet/dependencies.bicep new file mode 100644 index 00000000..e564f5c4 --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/kubenet/dependencies.bicep @@ -0,0 +1,27 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The name of the DNS Zone to create.') +param dnsZoneName string + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: managedIdentityName + location: location +} + +resource dnsZone 'Microsoft.Network/dnsZones@2018-05-01' = { + name: dnsZoneName + location: 'global' +} + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The resource ID of the created DNS Zone.') +output dnsZoneResourceId string = dnsZone.id diff --git a/bicep/modules/managed-cluster/tests/e2e/kubenet/main.test.bicep b/bicep/modules/managed-cluster/tests/e2e/kubenet/main.test.bicep new file mode 100644 index 00000000..6b5171e7 --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/kubenet/main.test.bicep @@ -0,0 +1,186 @@ +targetScope = 'subscription' + +metadata name = 'Using Kubenet Network Plugin.' +metadata description = 'This instance deploys the module with Kubenet network plugin .' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmkube' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + dnsZoneName: 'dep-${namePrefix}-dns-${serviceShort}.com' + location: resourceLocation + } +} + +// Diagnostics +// =========== +module diagnosticDependencies '../../../../../../utilities/e2e-template-assets/templates/diagnostic.dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-diagnosticDependencies' + params: { + storageAccountName: 'dep${namePrefix}diasa${serviceShort}01' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}' + eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + networkPlugin: 'kubenet' + diagnosticSettings: [ + { + name: 'customSetting' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + eventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + eventHubAuthorizationRuleResourceId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + storageAccountResourceId: diagnosticDependencies.outputs.storageAccountResourceId + workspaceResourceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + } + ] + roleAssignments: [ + { + name: '6acf186b-abbd-491b-8bd7-39fa199da81e' + roleDefinitionIdOrName: 'Owner' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + { + name: guid('Custom seed ${namePrefix}${serviceShort}') + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + { + roleDefinitionIdOrName: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'acdd72a7-3385-48ef-bd42-f606fba81ae7' + ) + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + ] + managedIdentities: { + userAssignedResourcesIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] + } + tags: { + 'hidden-title': 'This is visible in the resource name' + Environment: 'Non-Prod' + Role: 'DeploymentValidation' + } + } + dependsOn: [ + nestedDependencies + diagnosticDependencies + ] + } +] diff --git a/bicep/modules/managed-cluster/tests/e2e/priv/dependencies.bicep b/bicep/modules/managed-cluster/tests/e2e/priv/dependencies.bicep new file mode 100644 index 00000000..eab20c41 --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/priv/dependencies.bicep @@ -0,0 +1,93 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The Private DNS Zone Name to create for Private AKS Cluster.') +param privateDnsZoneName string + +@description('Required. The Name of the Virtual Network to create.') +param virtualNetworkName string + +var addressPrefix = '10.0.0.0/16' + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: managedIdentityName + location: location +} + +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: privateDnsZoneName + location: 'global' +} + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefix + ] + } + subnets: [ + { + name: 'defaultSubnet' + properties: { + addressPrefix: cidrSubnet(addressPrefix, 16, 0) + } + } + ] + } +} + +resource privateDNSZoneVNetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'pDnsLink-${virtualNetworkName}-${privateDnsZoneName}' + location: 'global' + parent: privateDnsZone + properties: { + registrationEnabled: true + virtualNetwork: { + id: virtualNetwork.id + } + } +} + +resource msiVnetRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, 'NetworkContributor', managedIdentity.id) + scope: virtualNetwork + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '4d97b98b-1d4f-4787-a291-c67834d212e7' + ) // Network Contributor + principalType: 'ServicePrincipal' + } +} + +resource msiPrivDnsZoneRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, 'PrivateDNSZoneContributor', managedIdentity.id) + scope: privateDnsZone + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'b12aa53e-6015-4669-85d0-8515ebb3ae7f' + ) // Private DNS Zone Contributor + principalType: 'ServicePrincipal' + } +} + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The resource ID of the private DNS Zone created.') +output privateDnsZoneResourceId string = privateDnsZone.id + +@description('The resource ID of the VirtualNetwork created.') +output vNetResourceId string = virtualNetwork.id diff --git a/bicep/modules/managed-cluster/tests/e2e/priv/main.test.bicep b/bicep/modules/managed-cluster/tests/e2e/priv/main.test.bicep new file mode 100644 index 00000000..94bf2b0d --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/priv/main.test.bicep @@ -0,0 +1,138 @@ +targetScope = 'subscription' + +metadata name = 'Using Private Cluster.' +metadata description = 'This instance deploys the module with a private cluster instance.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmpriv' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + privateDnsZoneName: 'privatelink.${resourceLocation}.azmk8s.io' + virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}' + location: resourceLocation + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + enablePrivateCluster: true + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + networkPlugin: 'azure' + skuTier: 'Standard' + dnsServiceIP: '10.10.200.10' + serviceCidr: '10.10.200.0/24' + privateDNSZone: nestedDependencies.outputs.privateDnsZoneResourceId + managedIdentities: { + userAssignedResourcesIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] + } + } + dependsOn: [ + nestedDependencies + ] + } +] diff --git a/bicep/modules/managed-cluster/tests/e2e/waf-aligned/dependencies.bicep b/bicep/modules/managed-cluster/tests/e2e/waf-aligned/dependencies.bicep new file mode 100644 index 00000000..eab20c41 --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/waf-aligned/dependencies.bicep @@ -0,0 +1,93 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The Private DNS Zone Name to create for Private AKS Cluster.') +param privateDnsZoneName string + +@description('Required. The Name of the Virtual Network to create.') +param virtualNetworkName string + +var addressPrefix = '10.0.0.0/16' + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: managedIdentityName + location: location +} + +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: privateDnsZoneName + location: 'global' +} + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefix + ] + } + subnets: [ + { + name: 'defaultSubnet' + properties: { + addressPrefix: cidrSubnet(addressPrefix, 16, 0) + } + } + ] + } +} + +resource privateDNSZoneVNetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'pDnsLink-${virtualNetworkName}-${privateDnsZoneName}' + location: 'global' + parent: privateDnsZone + properties: { + registrationEnabled: true + virtualNetwork: { + id: virtualNetwork.id + } + } +} + +resource msiVnetRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, 'NetworkContributor', managedIdentity.id) + scope: virtualNetwork + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '4d97b98b-1d4f-4787-a291-c67834d212e7' + ) // Network Contributor + principalType: 'ServicePrincipal' + } +} + +resource msiPrivDnsZoneRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, 'PrivateDNSZoneContributor', managedIdentity.id) + scope: privateDnsZone + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'b12aa53e-6015-4669-85d0-8515ebb3ae7f' + ) // Private DNS Zone Contributor + principalType: 'ServicePrincipal' + } +} + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The resource ID of the private DNS Zone created.') +output privateDnsZoneResourceId string = privateDnsZone.id + +@description('The resource ID of the VirtualNetwork created.') +output vNetResourceId string = virtualNetwork.id diff --git a/bicep/modules/managed-cluster/tests/e2e/waf-aligned/main.test.bicep b/bicep/modules/managed-cluster/tests/e2e/waf-aligned/main.test.bicep new file mode 100644 index 00000000..fb5bf064 --- /dev/null +++ b/bicep/modules/managed-cluster/tests/e2e/waf-aligned/main.test.bicep @@ -0,0 +1,196 @@ +targetScope = 'subscription' + +metadata name = 'WAF-aligned' +metadata description = 'This instance deploys the module in alignment with the best-practices of the Well-Architected Framework.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +// e.g., for a module 'network/private-endpoint' you could use 'dep-dev-network.privateendpoints-${serviceShort}-rg' +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +// e.g., for a module 'network/private-endpoint' you could use 'npe' as a prefix and then 'waf' as a suffix for the waf-aligned test +param serviceShort string = 'cswaf' + +@description('Optional. A token to inject into the name of each resource. This value can be automatically injected by the CI.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + privateDnsZoneName: 'privatelink.${resourceLocation}.azmk8s.io' + virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}' + location: resourceLocation + } +} + +// Diagnostics +// =========== +module diagnosticDependencies '../../../../../../utilities/e2e-template-assets/templates/diagnostic.dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-diagnosticDependencies' + params: { + storageAccountName: 'dep${namePrefix}diasa${serviceShort}01' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}' + eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}' + location: resourceLocation + } +} + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: resourceLocation +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + enablePrivateCluster: true + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + mode: 'System' + name: 'systempool' + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 0 + osType: 'Linux' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + osDiskType: 'Ephemeral' + osDiskSizeGB: 60 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + } + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 3 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + osDiskType: 'Ephemeral' + osDiskSizeGB: 60 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + networkPlugin: 'azure' + networkPolicy: 'azure' + skuTier: 'Standard' + dnsServiceIP: '10.10.200.10' + serviceCidr: '10.10.200.0/24' + omsAgentEnabled: true + monitoringWorkspaceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + disableLocalAccounts: true + enableAzureDefender: true + diagnosticSettings: [ + { + name: 'customSetting' + logCategoriesAndGroups: [ + { + category: 'kube-apiserver' + } + { + category: 'kube-controller-manager' + } + { + category: 'kube-scheduler' + } + { + category: 'cluster-autoscaler' + } + ] + metricCategories: [ + { + category: 'AllMetrics' + } + ] + eventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + eventHubAuthorizationRuleResourceId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + storageAccountResourceId: diagnosticDependencies.outputs.storageAccountResourceId + workspaceResourceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + } + ] + privateDNSZone: nestedDependencies.outputs.privateDnsZoneResourceId + managedIdentities: { + userAssignedResourcesIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] + } + tags: { + 'hidden-title': 'This is visible in the resource name' + Environment: 'Non-Prod' + Role: 'DeploymentValidation' + } + } + dependsOn: [ + nestedDependencies + diagnosticDependencies + ] + } +] diff --git a/bicep/modules/script-sshkeypair/version.json b/bicep/modules/managed-cluster/version.json similarity index 85% rename from bicep/modules/script-sshkeypair/version.json rename to bicep/modules/managed-cluster/version.json index 5c128522..76049e1c 100644 --- a/bicep/modules/script-sshkeypair/version.json +++ b/bicep/modules/managed-cluster/version.json @@ -1,6 +1,6 @@ { "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", - "version": "1.0", + "version": "0.3", "pathFilters": [ "./main.json" ] diff --git a/bicep/modules/script-sshkeypair/README.md b/bicep/modules/script-sshkeypair/README.md deleted file mode 100644 index 10878cf5..00000000 --- a/bicep/modules/script-sshkeypair/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# SSH Key Pair - -This module creates a SSH Key Pair and stores it in an Azure Key Vault - -## Details - -{{Add detailed information about the module}} - -## Parameters - -| Name | Type | Required | Description | -| :----------------------------------------- | :------: | :------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `kvName` | `string` | Yes | The name of the Azure Key Vault | -| `location` | `string` | No | The location of the Key Vault and where to deploy the module resources to | -| `forceUpdateTag` | `string` | No | How the deployment script should be forced to execute | -| `rbacRoleNeeded` | `string` | No | Azure RoleId that are required for the DeploymentScript resource to import images | -| `useExistingManagedIdentity` | `bool` | No | Does the Managed Identity already exists, or should be created | -| `managedIdentityName` | `string` | No | Name of the Managed Identity resource | -| `existingManagedIdentitySubId` | `string` | No | For an existing Managed Identity, the Subscription Id it is located in | -| `existingManagedIdentityResourceGroupName` | `string` | No | For an existing Managed Identity, the Resource Group it is located in | -| `sshKeyName` | `string` | Yes | The name of the SSH Key to be created.
if name is my-virtual-machine-ssh then the private key will be named my-virtual-machine-sshprivate and the public key will be named my-virtual-machine-sshpublic. | -| `initialScriptDelay` | `string` | No | A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate | -| `cleanupPreference` | `string` | No | When the script resource is cleaned up | - -## Outputs - -| Name | Type | Description | -| :--------------------- | :------: | :-------------------------------------------------- | -| `publicKeyUri` | `string` | The URI of the public key secret in the Key Vault | -| `privateKeyUri` | `string` | The URI of the private key secret in the Key Vault | -| `publicKeySecretName` | `string` | The name of the public key secret in the Key Vault | -| `privateKeySecretName` | `string` | The name of the private key secret in the Key Vault | - -## Examples - -### Example 1 - -```bicep -// Test with new managed identity -module test0 'br/public:deployment-scripts/aks-run-command:1.0.1' = { - name: 'test0-${uniqueString(name)}' - params: { - akvName: prereq.outputs.akvName - location: location - sshKeyName: 'first-key' - } -} - - -``` - -### Example 2 - -```bicep -// Test with existing managed identity -module test1 'br/public:deployment-scripts/aks-run-command:1.0.1' = { - dependsOn: [ - test0 - ] - name: 'test1-${uniqueString(name)}' - params: { - akvName: prereq.outputs.akvName - location: location - sshKeyName: 'second-key' - existingManagedIdentityResourceGroupName: resourceGroup().name - useExistingManagedIdentity: true - managedIdentityName: prereq.outputs.identityName - existingManagedIdentitySubId: subscription().subscriptionId - } -} -``` \ No newline at end of file diff --git a/bicep/modules/script-sshkeypair/metadata.json b/bicep/modules/script-sshkeypair/metadata.json deleted file mode 100644 index 8e6dc3fc..00000000 --- a/bicep/modules/script-sshkeypair/metadata.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$schema": "https://aka.ms/bicep-registry-module-metadata-file-schema-v2#", - "name": "SSH Key Pair Creation", - "summary": "Creates SSH Key Pair Stores them in KeyVault as Secrets", - "owner": "mci-get-energy" -} \ No newline at end of file diff --git a/bicep/modules/script-sshkeypair/script.sh b/bicep/modules/script-sshkeypair/script.sh deleted file mode 100644 index bb7d2fc0..00000000 --- a/bicep/modules/script-sshkeypair/script.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -e - -echo "Waiting on Identity RBAC replication ($initialDelay)" -sleep $initialDelay - -# Generate the SSH key pair -echo "Generating SSH key pair..." -#ssh-keygen -t rsa -b 4096 -C "azure@example.com" -f id_rsa -N "" -ssh-keygen -m PEM -t rsa -b 4096 -f id_rsa -q - -# Import the private key and public key as strings -privateKey=$(cat id_rsa) -publicKey=$(cat id_rsa.pub) - -# Re-Login to Azure CLI using the managed identity -echo "Logging in to Azure CLI using managed identity..." -az login --identity - -echo "Storing secret ${sshKeyName}private in Key Vault $keyVaultName..." -privSecret=$(az keyvault secret set --vault-name "$keyVaultName" --name "${sshKeyNamePrivate}" --value "$privateKey") - -echo "Storing secret ${sshKeyName}public in Key Vault $keyVaultName..." -pubSecret=$(az keyvault secret set --vault-name "$keyVaultName" --name "${sshKeyNamePublic}" --value "$publicKey") - -privateSecretId=$(echo $privSecret | jq -r ".id" | cut -d'/' -f-5) # remove the version from the url; -publicSecretId=$(echo $pubSecret | jq -r ".id" | cut -d'/' -f-5) # remove the version from the url; - -jsonOutputString=$(jq -cn --arg public $publicSecretId --arg private $privateSecretId '{secretUris: $ARGS.named}') -echo $jsonOutputString -echo $jsonOutputString > $AZ_SCRIPTS_OUTPUT_PATH - -# Cleanup -rm -f id_rsa id_rsa.pub \ No newline at end of file diff --git a/bicep/modules/script-sshkeypair/test/main.test.bicep b/bicep/modules/script-sshkeypair/test/main.test.bicep deleted file mode 100644 index a544e3d9..00000000 --- a/bicep/modules/script-sshkeypair/test/main.test.bicep +++ /dev/null @@ -1,67 +0,0 @@ -targetScope = 'resourceGroup' - -@minLength(3) -@maxLength(10) -@description('Used to name all resources') -param resourceName string - -@description('Registry Location.') -param location string = resourceGroup().location - -//Prerequisites -module identity '../../user-managed-identity/main.bicep' = { - name: 'user-managed-identity' - params: { - resourceName: resourceName - location: location - } -} - -module kv '../../azure-keyvault/main.bicep' = { - name: 'azure_keyvault' - params: { - resourceName: resourceName - location: location - secretsObject: { secrets: [] } - - // Add Role Assignment - roleAssignments: [ - { - roleDefinitionIdOrName: 'Key Vault Administrator' - principals: [ - { - id: identity.outputs.principalId - } - ] - principalType: 'ServicePrincipal' - } - ] - } -} - -// Test with new managed identity -module test0 '../main.bicep' = { - name: 'test0-${uniqueString(resourceName)}' - params: { - kvName: kv.outputs.name - location: location - sshKeyName: 'first-key' - } -} - -// Test with existing managed identity -module test1 '../main.bicep' = { - dependsOn: [ - test0 - ] - name: 'test1-${uniqueString(resourceName)}' - params: { - kvName: kv.outputs.name - location: location - sshKeyName: 'second-key' - existingManagedIdentityResourceGroupName: resourceGroup().name - useExistingManagedIdentity: true - managedIdentityName: identity.outputs.name - existingManagedIdentitySubId: subscription().subscriptionId - } -} diff --git a/bicep/modules/script-sshkeypair/test/parameters.json b/bicep/modules/script-sshkeypair/test/parameters.json deleted file mode 100644 index 4d4c4d1d..00000000 --- a/bicep/modules/script-sshkeypair/test/parameters.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "template": "../main.bicep" - }, - "parameters": { - "location": { - "value": "southcentralus" - }, - "resourceName": { - "value": "adme" - } - } -} \ No newline at end of file diff --git a/bicep/modules/script-sshkeypair/main.bicep b/bicep/modules/software-upload/main.bicep similarity index 51% rename from bicep/modules/script-sshkeypair/main.bicep rename to bicep/modules/software-upload/main.bicep index 78f4a041..307a012b 100644 --- a/bicep/modules/script-sshkeypair/main.bicep +++ b/bicep/modules/software-upload/main.bicep @@ -1,24 +1,39 @@ -metadata name = 'SSH Key Pair' -metadata description = 'This module creates a SSH Key Pair and stores it in an Azure Key Vault' -metadata owner = 'azure-global-energy' +metadata name = 'Blob Upload' +metadata description = 'This module uploads a file to a blob storage account' +metadata owner = 'daniel-scholl' -@description('The name of the Azure Key Vault') -param kvName string +@description('Desired name of the storage account') +param storageAccountName string = uniqueString(resourceGroup().id, deployment().name, 'blob') -@description('The location of the Key Vault and where to deploy the module resources to') +@description('Name of the container') +param containerName string = 'gitops' + +@description('Name of the file as it is stored in the share') +param filename string = 'main.zip' + +@description('Name of the directory to upload') +param directoryName string + +@description('The source of the software to upload') +param softwareSource string = 'https://github.com/azure/osdu-developer' + +@description('Name of the file as it is stored in the share') +param fileurl string = '/archive/refs/heads/main.zip' + +@description('The location of the Storage Account and where to deploy the module resources to') param location string = resourceGroup().location @description('How the deployment script should be forced to execute') param forceUpdateTag string = utcNow() -@description('Azure RoleId that are required for the DeploymentScript resource to import images') -param rbacRoleNeeded string = 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7' //Key Vault Secrets officer is needed to create secrets in the Key Vault +@description('Azure RoleId that are required for the DeploymentScript resource to upload blobs') +param rbacRoleNeeded string = '' //Storage File Contributor is needed to upload @description('Does the Managed Identity already exists, or should be created') param useExistingManagedIdentity bool = false @description('Name of the Managed Identity resource') -param managedIdentityName string = 'id-keyvault-ssh-${location}' +param managedIdentityName string = 'id-storage-share-${location}' @description('For an existing Managed Identity, the Subscription Id it is located in') param existingManagedIdentitySubId string = subscription().subscriptionId @@ -26,12 +41,6 @@ param existingManagedIdentitySubId string = subscription().subscriptionId @description('For an existing Managed Identity, the Resource Group it is located in') param existingManagedIdentityResourceGroupName string = resourceGroup().name -@description(''' -The name of the SSH Key to be created. -if name is my-virtual-machine-ssh then the private key will be named my-virtual-machine-sshprivate and the public key will be named my-virtual-machine-sshpublic. -''') -param sshKeyName string - @description('A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate') param initialScriptDelay string = '30s' @@ -39,26 +48,24 @@ param initialScriptDelay string = '30s' @description('When the script resource is cleaned up') param cleanupPreference string = 'OnSuccess' -var privateKeySecretName = '${sshKeyName}private' -var publicKeySecretName = '${sshKeyName}public' -resource akv 'Microsoft.KeyVault/vaults@2022-11-01' existing = { - name: kvName +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = { + name: storageAccountName } -resource newDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (!useExistingManagedIdentity) { +resource newDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-07-31-preview' = if (!useExistingManagedIdentity) { name: managedIdentityName location: location } -resource existingDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (useExistingManagedIdentity) { +resource existingDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-07-31-preview' existing = if (useExistingManagedIdentity) { name: managedIdentityName scope: resourceGroup(existingManagedIdentitySubId, existingManagedIdentityResourceGroupName) } resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(rbacRoleNeeded)) { - name: guid(akv.id, rbacRoleNeeded, useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id) - scope: akv + name: guid(storageAccount.id, rbacRoleNeeded, useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id) + scope: storageAccount properties: { roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', rbacRoleNeeded) principalId: useExistingManagedIdentity ? existingDepScriptId.properties.principalId : newDepScriptId.properties.principalId @@ -66,8 +73,8 @@ resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty( } } -resource createSshKeyPair 'Microsoft.Resources/deploymentScripts@2023-08-01' = { - name: 'script-${akv.name}-${replace(replace(sshKeyName, ':', ''), '/', '-')}' +resource uploadFile 'Microsoft.Resources/deploymentScripts@2023-08-01' = { + name: 'script-${storageAccount.name}-${replace(replace(filename, ':', ''), '/', '-')}' location: location identity: { type: 'UserAssigned' @@ -81,9 +88,12 @@ resource createSshKeyPair 'Microsoft.Resources/deploymentScripts@2023-08-01' = { timeout: 'PT30M' retentionInterval: 'PT1H' environmentVariables: [ - { name: 'keyVaultName', value: akv.name } - { name: 'sshKeyNamePrivate', value: privateKeySecretName } - { name: 'sshKeyNamePublic', value: publicKeySecretName } + { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } + { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } + { name: 'FILE', value: filename } + { name: 'URL', value: '${softwareSource}${fileurl}' } + { name: 'CONTAINER', value: containerName } + { name: 'UPLOAD_DIR', value: string(directoryName) } { name: 'initialDelay', value: initialScriptDelay } ] scriptContent: loadTextContent('script.sh') @@ -91,12 +101,3 @@ resource createSshKeyPair 'Microsoft.Resources/deploymentScripts@2023-08-01' = { } } -@description('The URI of the public key secret in the Key Vault') -output publicKeyUri string = createSshKeyPair.properties.outputs.secretUris.public -@description('The URI of the private key secret in the Key Vault') -output privateKeyUri string = createSshKeyPair.properties.outputs.secretUris.private - -@description('The name of the public key secret in the Key Vault') -output publicKeySecretName string = publicKeySecretName -@description('The name of the private key secret in the Key Vault') -output privateKeySecretName string = privateKeySecretName diff --git a/bicep/modules/software-upload/script.sh b/bicep/modules/software-upload/script.sh new file mode 100644 index 00000000..d5500b00 --- /dev/null +++ b/bicep/modules/software-upload/script.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -e + +echo "Waiting on Identity RBAC replication (${initialDelay})" +sleep ${initialDelay} + +# Installing required packages +apk add --no-cache curl zip unzip + +# Download the file using curl +echo "Downloading file from ${URL}" +curl -L -o repo.zip "${URL}" + +# Check if the download was successful +if [ $? -ne 0 ]; then + echo "Failed to download the file from ${URL}" + exit 1 +fi + +# Create a directory for extracted files +mkdir -p extracted_files + +# Unzip the file +echo "Extracting contents..." +unzip -q repo.zip -d extracted_files + +# Find and replace 'kind: GitRepository' with 'kind: Bucket' in all files +find extracted_files -type f -path "*/${UPLOAD_DIR}/*" -exec sed -i ' + /sourceRef:/{ + N;N;N + s/sourceRef:\n[[:space:]]*kind: GitRepository\n[[:space:]]*name: flux-system\n[[:space:]]*namespace: flux-system/sourceRef:\n kind: Bucket\n name: flux-system\n namespace: flux-system/g + }' {} + + +# Find the software directory +software_dir=$(find extracted_files -type d -name "${UPLOAD_DIR}" -exec dirname {} \;) + +if [ -z "$software_dir" ]; then + echo "Error: '${UPLOAD_DIR}' directory not found in the extracted contents." + exit 1 +fi + +# Upload the contents of the software directory +echo "Uploading files from ${software_dir} to blob container ${CONTAINER}" +az storage blob upload-batch --destination ${CONTAINER} --source "${software_dir}" --pattern "${UPLOAD_DIR}/**" --overwrite true --auth-mode login +echo "Files from software directory uploaded to blob container ${CONTAINER}." + +# Clean up +rm -rf extracted_files repo.zip diff --git a/bicep/modules/virtual_machine.bicep b/bicep/modules/virtual_machine.bicep deleted file mode 100644 index dbeda612..00000000 --- a/bicep/modules/virtual_machine.bicep +++ /dev/null @@ -1,177 +0,0 @@ -// Parameters -@description('Specifies the name of the virtual machine.') -param vmName string = 'simpleVM' - -@description('Specifies the size of the virtual machine.') -param vmSize string = 'Standard_D2s_v3' - -@description('Specifies the resource id of the subnet hosting the virtual machine.') -param vmSubnetId string - -@description('Specifies the image publisher of the disk image used to create the virtual machine.') -param imagePublisher string = 'Canonical' - -@description('Specifies the offer of the platform image or marketplace image used to create the virtual machine.') -param imageOffer string = '0001-com-ubuntu-server-jammy' - -@description('Specifies the Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version.') -param imageSku string = '22_04-lts-gen2' - -@description('Specifies the type of authentication when accessing the Virtual Machine. SSH key is recommended.') -@allowed([ - 'sshPublicKey' - 'password' -]) -param authenticationType string = 'password' - -@description('Specifies the name of the administrator account of the virtual machine.') -param vmAdminUsername string - -@description('Specifies the SSH Key or password for the virtual machine. SSH key is recommended.') -@secure() -param vmAdminPasswordOrKey string - -@description('Specifies the storage account type for OS and data disk.') -@allowed([ - 'Premium_LRS' - 'StandardSSD_LRS' - 'Standard_LRS' - 'UltraSSD_LRS' -]) -param diskStorageAccountType string = 'Standard_LRS' - -@description('Specifies the number of data disks of the virtual machine.') -@minValue(0) -@maxValue(64) -param numDataDisks int = 0 - -@description('Specifies the size in GB of the OS disk of the VM.') -param osDiskSize int = 30 - -@description('Specifies the size in GB of the OS disk of the virtual machine.') -param dataDiskSize int = 50 - -@description('Specifies the caching requirements for the data disks.') -param dataDiskCaching string = 'ReadWrite' - -@description('Specifies the name of the Log Analytics workspace.') -param workspaceName string - -@description('Specifies the location.') -param location string = resourceGroup().location - -@description('Specifies the resource tags.') -param tags object - - -// Variables -var vmNicName = '${vmName}Nic' -var linuxConfiguration = { - disablePasswordAuthentication: true - ssh: { - publicKeys: [ - { - path: '/home/${vmAdminUsername}/.ssh/authorized_keys' - keyData: vmAdminPasswordOrKey - } - ] - } - provisionVMAgent: true -} - - -// Resources -resource virtualMachineNic 'Microsoft.Network/networkInterfaces@2023-06-01' = { - name: vmNicName - location: location - tags: tags - properties: { - ipConfigurations: [ - { - name: 'ipconfig1' - properties: { - privateIPAllocationMethod: 'Dynamic' - subnet: { - id: vmSubnetId - } - } - } - ] - } -} - - - -resource virtualMachine 'Microsoft.Compute/virtualMachines@2023-09-01' = { - name: vmName - location: location - tags: tags - properties: { - hardwareProfile: { - vmSize: vmSize - } - osProfile: { - computerName: vmName - adminUsername: vmAdminUsername - adminPassword: vmAdminPasswordOrKey - linuxConfiguration: (authenticationType == 'password') ? null : linuxConfiguration - } - storageProfile: { - imageReference: { - publisher: imagePublisher - offer: imageOffer - sku: imageSku - version: 'latest' - } - osDisk: { - name: '${vmName}_OSDisk' - caching: 'ReadWrite' - createOption: 'FromImage' - diskSizeGB: osDiskSize - managedDisk: { - storageAccountType: diskStorageAccountType - } - } - dataDisks: [for j in range(0, numDataDisks): { - caching: dataDiskCaching - diskSizeGB: dataDiskSize - lun: j - name: '${vmName}-DataDisk${j}' - createOption: 'Empty' - managedDisk: { - storageAccountType: diskStorageAccountType - } - }] - } - networkProfile: { - networkInterfaces: [ - { - id: virtualMachineNic.id - } - ] - } - } -} - -resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { - name: workspaceName -} - -resource omsAgentForLinux 'Microsoft.Compute/virtualMachines/extensions@2023-09-01' = { - parent: virtualMachine - name: 'LogAnalytics' - location: location - properties: { - publisher: 'Microsoft.EnterpriseCloud.Monitoring' - type: 'OmsAgentForLinux' - typeHandlerVersion: '1.17' - settings: { - workspaceId: logAnalyticsWorkspace.properties.customerId - stopOnMultipleConnections: false - } - protectedSettings: { - workspaceKey: logAnalyticsWorkspace.listKeys().primarySharedKey //listKeys(logAnalyticsWorkspace.id, logAnalyticsWorkspace.apiVersion).primarySharedKey - } - } -} - diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 5b190a13..c2f57abc 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -79,6 +79,8 @@ markdown_extensions: - pymdownx.snippets - pymdownx.tabbed: alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true - pymdownx.superfences: custom_fences: - name: mermaid @@ -141,6 +143,7 @@ nav: - experimental_adminui.md - Design: - design_architecture.md + - design_platform.md - design_infrastructure.md - design_software.md - Tutorials: diff --git a/docs/src/advanced_vnet.md b/docs/src/advanced_vnet.md index e32fea29..91b9badb 100644 --- a/docs/src/advanced_vnet.md +++ b/docs/src/advanced_vnet.md @@ -216,7 +216,7 @@ Use the following commands set up the network with a required subnet for the clu --query "id" -o tsv) # role_assignment - az role assignment create --assignee $IDENTITY_ID \ + az role assignment create --assignee $IDENTITY_PID \ --role "Network Contributor" \ --scope $NETWORK_ID ``` diff --git a/docs/src/design_platform.md b/docs/src/design_platform.md new file mode 100644 index 00000000..54e0e50d --- /dev/null +++ b/docs/src/design_platform.md @@ -0,0 +1,192 @@ +# Platform + +The OSDU™ private instance solution implements industry-leading best practices for security and operational excellence on Azure Kubernetes Service (AKS). These practices are aligned with Microsoft's Secure Future Initiative and are designed to provide a robust, secure, and efficient platform while maintaining developer productivity. + +??? Tip "Learning Opportunity" + For more details on Microsoft's security focus, refer to the + [Microsoft Secure Future Initiative](https://www.microsoft.com/security/business/secure-future-initiative). + +This solution implements comprehensive best practices across security controls and operational excellence. The implemented controls and features help ensure: + +- Strong security posture through infrastructure and application security controls +- Operational efficiency through automation and DevOps practices +- Reliable performance through proper scaling and maintenance procedures +- Simplified maintenance through automated updates and proper backup strategies + +??? Tip "Learning Opportunity" + For more details on Microsoft's Cluster Best Practices, refer to the + [AKS Best Practices](https://learn.microsoft.com/en-us/azure/aks/best-practices). + +## Security Controls + +### Infrastructure Security + +

+ +- :material-shield-check:{ .lg .middle } __Cluster Protection__ + + --- + + - [x] [Microsoft Defender for Containers](https://learn.microsoft.com/en-us/azure/defender-for-cloud/defender-for-containers-introduction) + + Comprehensive security monitoring and protection for containerized assets including clusters, nodes, workloads, registries and images. + + - [x] [Kubernetes RBAC and Microsoft Entra ID](https://learn.microsoft.com/en-us/azure/aks/concepts-identity) + + Granular access control by granting users, groups, and service accounts only the minimum required permissions through role-based policies and enhanced Azure authentication. + + - [x] [Node Resource Group Lockdown](https://learn.microsoft.com/en-us/azure/aks/node-resource-group-lockdown) + + Prevent unauthorized changes to node resource group resources using NRGLockdownPreview feature. + +
+ +
+ +- :material-linux:{ .lg .middle } __Node Security__ + + --- + + - [x] [Azure Linux](https://learn.microsoft.com/en-us/azure/aks/use-azure-linux) + + Azure Linux Container Host is optimized for container workloads on AKS, based on Microsoft's CBL-Mariner Linux distribution. + + - [x] [Disable SSH Access](https://learn.microsoft.com/en-us/azure/aks/disable-ssh-access) + + Improve security by disabling SSH access to nodes at both cluster and node pool levels using DisableSSHPreview feature. + +
+ +
+ +- :material-network:{ .lg .middle } __Network Security__ + + --- + + - [x] [CNI Overlay](https://learn.microsoft.com/en-us/azure/aks/azure-cni-overlay) + + Enhanced network security with overlay networking, providing logical separation between pod and node networks. + + - [x] [NAT Gateway](https://learn.microsoft.com/en-us/azure/aks/nat-gateway) + + Managed outbound internet connectivity with network isolation capabilities. + + - [x] [Service Mesh](https://learn.microsoft.com/en-us/azure/aks/istio-deploy-addon) + + Istio service mesh for secure service-to-service communication, traffic management, and observability. + +
+ +
+ +- :material-database:{ .lg .middle } __Storage Security__ + + --- + + - [x] [Managed Disks](https://learn.microsoft.com/en-us/azure/aks/azure-disk-customer-managed-keys) + + Secure block-level storage volumes with encryption and access controls. + +
+ +### Application Security + +
+ +- :material-docker:{ .lg .middle } __Container Security__ + + --- + + - [x] [Image Cleaner](https://learn.microsoft.com/en-us/azure/aks/image-cleaner) + + Automatic identification and removal of unused images to reduce vulnerability surface. + +
+ +
+ +- :material-shield-lock:{ .lg .middle } __Pod Security__ + + --- + + - [x] [Pod Security Context](https://learn.microsoft.com/en-us/azure/aks/developer-best-practices-pod-security) + + Limit access to processes and services through security context settings, implementing principle of least privilege. + + - [x] [Workload Identity](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview) + + Enable pods to authenticate against Azure services using Microsoft Entra workload identities. + + - [x] [Secrets Management](https://learn.microsoft.com/en-us/azure/aks/csi-secrets-store-driver) + + Integrate Azure Key Vault with Secrets Store CSI Driver for secure runtime secrets management. + + - [x] [Policy Controls](https://learn.microsoft.com/en-us/azure/aks/policy-reference) + + Enforce Kubernetes best practices through Azure Policy deployment safeguards. + +
+ +## Operational Excellence + +### Automation & DevOps + +
+ +- :material-cog:{ .lg .middle } __Deployment & Operations__ + + --- + + - [x] [GitOps](https://learn.microsoft.com/en-us/azure/azure-arc/kubernetes/tutorial-use-gitops-flux2) + + Git-based infrastructure and application deployment management. + + - [x] [Verified Modules](https://learn.microsoft.com/en-us/azure/verified-modules/overview) + + Pre-validated infrastructure modules for consistent and secure deployments. + + - [x] [App Configuration](https://learn.microsoft.com/en-us/azure/azure-app-configuration/overview) + + Managed service for feature flags and configuration management. + +
+ +### Scalability & Performance + +
+ +- :material-speedometer:{ .lg .middle } __Performance & Scaling__ + + --- + + - [x] [Node Auto Provisioning](https://learn.microsoft.com/en-us/azure/aks/cluster-node-auto-provisioning) + + Automatic node provisioning for optimal cluster sizing and cost efficiency. + + - [x] [KEDA](https://learn.microsoft.com/en-us/azure/aks/keda-about) + + Event-driven autoscaling for Kubernetes workloads. + + - [x] [Vertical Pod Autoscaler](https://learn.microsoft.com/en-us/azure/aks/vertical-pod-autoscaler) + + Automated resource allocation optimization for pods based on usage patterns. + +
+ +### Maintenance & Updates + +
+ +- :material-update:{ .lg .middle } __System Updates__ + + --- + + - [x] [Automatic Upgrades](https://learn.microsoft.com/en-us/azure/aks/auto-upgrade-cluster?tabs=azure-cli) + + Stay current on new features and bug fixes with automated Kubernetes version upgrades. + + - [x] [Node OS Updates](https://learn.microsoft.com/en-us/azure/aks/node-updates-kured) + + Linux nodes in AKS get security patches through their distro update channel nightly. + +
diff --git a/docs/src/feature_flags.md b/docs/src/feature_flags.md index df1645b4..3c30dd74 100644 --- a/docs/src/feature_flags.md +++ b/docs/src/feature_flags.md @@ -13,10 +13,10 @@ Feature flags are configuration settings used to modify the default behavior of !!! warning "CLI Deployment Only" - Feature flags are implemented as named environment variables which correspond to ARM template parameter objects. + Feature flags are implemented as named environment variables which correspond to ARM template parameter objects and are set prior to provisioning. -!!! tip "Setting Feature Flags" - Set feature flags prior to provisioning. +=== "Command" + ```bash azd env set ``` @@ -48,12 +48,9 @@ Infrastructure customizations can be modified using the following feature flags. | Feature Flag | Description | |---------------------------|-----------------------------------------------------------------------------| -| ENABLE_BURSTABLE | User cheaper Burstable server types in the cluster | | CLUSTER_INGRESS | Specifies the Ingress type for the cluster (External, Internal, or Both) | | CLUSTER_VM_SIZE | Overrides the default server type with a custom VM size | | ENABLE_BLOB_PUBLIC_ACCESS | Enables public access for storage account blob (False by default) | -| ENABLE_MANAGE | Enables a Bastion Host with a virtual machine for private admin access | - ## Custom Software diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md index 00ba1e98..73439520 100644 --- a/docs/src/getting_started.md +++ b/docs/src/getting_started.md @@ -4,19 +4,21 @@ Prerequisites and configuration steps for deploying personal OSDU™ instances i ## Subscription Quota -It is recommended to have at least 50 vCPUs in a region along with the ability to deploy Cosmos DB instances which can be resource constrained in some regions. Defaults can be increased by requesting a [quota increase](https://learn.microsoft.com/en-us/azure/quotas/regional-quota-requests). +It is recommended to have at least 50 vCPUs in a region for vCPU families along with the ability to deploy Cosmos DB instances which can be resource constrained in some regions. Defaults can be increased by requesting a [quota increase](https://learn.microsoft.com/en-us/azure/quotas/regional-quota-requests). !!! note "Ensure Sufficient Quota" - The choice between BS and DS family vCPUs depends on your specific deployment requirements: + The deployment requires quota for the following VM families: + + - Standard_D4pds_v5 nodes for system workloads + - Standard_D2pds_v5 nodes for zonal workloads + - Standard_D4s_v5 nodes for default workloads - - Increase DS family vCPU quota if necessary. - - Increase BS family vCPU quota if using `ENABLE_BATCH`. | Quota Name | Minimum Quantity | |------------|------------------| | Total Regional vCPUs | 100 | -| Standard BS Family vCPUs | 50 | -| Standard DS Family vCPUs | 50 | +| Standard DPDSv5 Family vCPUs | 50 | +| Standard DSv5 Family vCPUs | 50 | !!! tip "Available Cosmos DB Regions" diff --git a/docs/src/install_portal.md b/docs/src/install_portal.md index 00303219..0d4579ca 100644 --- a/docs/src/install_portal.md +++ b/docs/src/install_portal.md @@ -42,7 +42,6 @@ The Azure Resource Manager (ARM) custom template deployment provides a simple wa 4. Modify the optional parameters as desired. === "Optional Parameters" - - Enable Burstable: _`Feature Flag: Enable burstable server types.`_ - Custom VM Size: _`Set Custom VM size cluster nodes.`_ - Ingress Type: _`Switch: Ingress type to use.`_ - Enable Blob Public Access: _`Feature Flag: Enable Blob Storage public access.`_ diff --git a/software/components/elastic-search/elastic-search.yaml b/software/components/elastic-search/elastic-search.yaml index 9ae2491a..036bca82 100644 --- a/software/components/elastic-search/elastic-search.yaml +++ b/software/components/elastic-search/elastic-search.yaml @@ -72,9 +72,10 @@ spec: fieldPath: metadata.annotations['topology.kubernetes.io/zone'] resources: requests: + memory: 2Gi cpu: 0.2 limits: - memory: 8Gi + memory: 2Gi cpu: 2 topologySpreadConstraints: - maxSkew: 1