From 9379e12871c418d9e0fd1977e4ea76bd487c4e32 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 3 Nov 2024 11:36:31 -0600 Subject: [PATCH] Security Changes (#226) * Enabled Node Auto Provisioning and Private Cluster. * Removed localhost from post provision. * Added software upload --- README.md | 8 +++++ bicep/main.bicep | 48 ++++++++++++++------------ bicep/main.parameters.json | 6 ++++ bicep/modules/blade_common.bicep | 57 ++++++++++++++++--------------- bicep/modules/blade_service.bicep | 48 ++++++++++++++++---------- docs/src/design_platform.md | 4 +-- docs/src/getting_started.md | 30 ++++++++++++++++ scripts/post-provision.ps1 | 2 +- 8 files changed, 132 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 8a0b1daf..80ae7382 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,14 @@ This project provides simplified personal deployments of the [OSDU™](https://c ### Getting Started +This project uses the following Azure Container Service preview features: + +- [API Server VNet Integration](https://learn.microsoft.com/en-us/azure/aks/api-server-vnet-integration) +- [Node Resource Group Lockdown](https://learn.microsoft.com/en-us/azure/aks/node-resource-group-lockdown) +- [AKS Safeguards](https://learn.microsoft.com/en-us/azure/aks/deployment-safeguards) +- [Node Auto Provisioning](https://learn.microsoft.com/en-us/azure/aks/node-autoprovision?tabs=azure-cli) +- [SSH Disable](https://learn.microsoft.com/en-us/azure/aks/manage-ssh-node-access?tabs=node-shell#disable-ssh-overview) + Review the [Getting Started](https://azure.github.io/osdu-developer/getting_started/) documentation prior to proceeding. ### CLI Quickstart diff --git a/bicep/main.bicep b/bicep/main.bicep index 069d109f..8dd5180e 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -35,7 +35,29 @@ param enableBlobPublicAccess bool = false @description('Feature Flag: Enable AKS Enhanced Subnet Support (Azure CNI)') param enablePodSubnet bool = false -// This would be a type but bugs exist for ARM Templates so is object instead. +@description('Optional: Cluster Configuration Overrides') +param clusterConfiguration object = { + enablePrivateCluster: '' + enableNodeAutoProvisioning: '' +} + +@description('(Optional) Software Load Override - {enable/osduCore/osduReference} --> true/false, {repository} --> https://github.com/azure/osdu-devloper {branch} --> branch:main') +param clusterSoftware object = { + enable: true + osduCore: true + osduReference: true + osduVersion: '' + repository: '' + branch: '' + tag: '' +} + +@description('(Optional) Experimental Software Override - {enable/adminUI} --> true/false') +param experimentalSoftware object = { + enable: false + adminUI: false +} + @description('Optional. Bring your own Virtual Network.') param vnetConfiguration object = { group: '' @@ -60,26 +82,6 @@ param vnetConfiguration object = { } } - - -@description('(Optional) Software Load Override - {enable/osduCore/osduReference} --> true/false, {repository} --> https://github.com/azure/osdu-devloper {branch} --> branch:main') -param clusterSoftware object = { - enable: true - osduCore: true - osduReference: true - osduVersion: '' - repository: '' - branch: '' - tag: '' -} - -@description('(Optional) Experimental Software Override - {enable/adminUI} --> true/false') -param experimentalSoftware object = { - enable: false - adminUI: false -} - - // This would be a type but bugs exist for ARM Templates so is object instead. @description('Cluster Network Overrides - {ingress} (Both/Internal/External), {serviceCidr}, {dnsServiceIP}') param clusterNetwork object = { @@ -114,7 +116,6 @@ var cmekConfiguration = { // <- Internal Feature Flags End - @description('Internal Configuration Object') var configuration = { name: 'main' @@ -344,6 +345,9 @@ module serviceBlade 'modules/blade_service.bicep' = { enableOsduCore: clusterSoftware.osduCore == 'false' ? false : true enableOsdureference: clusterSoftware.osduReference == 'false' ? false : true + enableNodeAutoProvisioning: clusterConfiguration.enableNodeAutoProvisioning == 'false' ? false : true + enablePrivateCluster: clusterConfiguration.enablePrivateCluster == 'true' ? true : false + enableExperimental: experimentalSoftware.enable == 'true' ? true : false enableAdminUI: experimentalSoftware.adminUI == 'true' ? true : false diff --git a/bicep/main.parameters.json b/bicep/main.parameters.json index 40731927..71468ed1 100644 --- a/bicep/main.parameters.json +++ b/bicep/main.parameters.json @@ -29,6 +29,12 @@ "enableBlobPublicAccess": { "value": "${ENABLE_BLOB_PUBLIC_ACCESS}" }, + "clusterConfiguration": { + "value": { + "enableNodeAutoProvisioning": "${ENABLE_NODE_AUTO_PROVISIONING}", + "enablePrivateCluster": "${ENABLE_PRIVATE_CLUSTER}" + } + }, "vnetConfiguration": { "value": { "group": "${VIRTUAL_NETWORK_GROUP}", diff --git a/bicep/modules/blade_common.bicep b/bicep/modules/blade_common.bicep index 050895f5..bd932564 100644 --- a/bicep/modules/blade_common.bicep +++ b/bicep/modules/blade_common.bicep @@ -324,34 +324,35 @@ module configStorage './storage-account/main.bicep' = { } } -// var directoryUploads = [ -// { -// directory: 'software' -// } -// { -// directory: 'charts' -// } -// { -// directory: 'stamp' -// } -// ] - -// @batchSize(1) -// module gitOpsUpload './software-upload/main.bicep' = [for item in directoryUploads: { -// name: '${bladeConfig.sectionName}-storage-${item.directory}-upload' -// params: { -// storageAccountName: configStorage.outputs.name -// location: location -// useExistingManagedIdentity: true -// managedIdentityName: userAssignedIdentity.name -// existingManagedIdentitySubId: subscription().subscriptionId -// existingManagedIdentityResourceGroupName: resourceGroup().name -// directoryName: item.directory -// } -// dependsOn: [ -// configStorage -// ] -// }] +var directoryUploads = [ + { + directory: 'software' + } + { + directory: 'charts' + } + { + directory: 'stamp' + } +] + +@batchSize(1) +module gitOpsUpload './software-upload/main.bicep' = [for item in directoryUploads: { + name: '${bladeConfig.sectionName}-storage-${item.directory}-upload' + params: { + storageAccountName: configStorage.outputs.name + location: location + useExistingManagedIdentity: true + managedIdentityName: userAssignedIdentity.name + existingManagedIdentitySubId: subscription().subscriptionId + existingManagedIdentityResourceGroupName: resourceGroup().name + directoryName: item.directory + rbacRoleNeeded: 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' // Storage Blob Data Owner + } + dependsOn: [ + configStorage + ] +}] resource storageDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (enablePrivateLink) { name: storageDnsZoneName diff --git a/bicep/modules/blade_service.bicep b/bicep/modules/blade_service.bicep index b3089fdc..1096a51d 100644 --- a/bicep/modules/blade_service.bicep +++ b/bicep/modules/blade_service.bicep @@ -121,6 +121,13 @@ param appSettings appConfigItem[] param dateStamp string = utcNow() + +@description('Feature Flag to Enable Node Auto Provisioning') +param enableNodeAutoProvisioning bool = true + +@description('Feature Flag to Enable Private Cluster') +param enablePrivateCluster bool = true + ///////////////////////////////// // Configuration ///////////////////////////////// @@ -295,7 +302,7 @@ module cluster './managed-cluster/main.bicep' = { networkDataplane: 'cilium' publicNetworkAccess: 'Enabled' outboundType: empty(aksSubnetId) ? 'managedNATGateway' : 'loadBalancer' - enablePrivateCluster: false + enablePrivateCluster: enablePrivateCluster // Access Settings disableLocalAccounts: true @@ -326,7 +333,7 @@ module cluster './managed-cluster/main.bicep' = { enableKeyvaultSecretsProvider: true enableSecretRotation: true enableImageCleaner: true - imageCleanerIntervalHours: 168 + imageCleanerIntervalHours: 24 enableOidcIssuerProfile: true enableWorkloadIdentity: true azurePolicyEnabled: true @@ -335,7 +342,7 @@ module cluster './managed-cluster/main.bicep' = { // Auto-Scaling vpaAddon: true kedaAddon: true - enableNodeAutoProvisioning: false + enableNodeAutoProvisioning: enableNodeAutoProvisioning maintenanceConfiguration: { maintenanceWindow: { @@ -360,9 +367,10 @@ module cluster './managed-cluster/main.bicep' = { name: 'system' mode: 'System' vmSize: empty(vmSize) ? serviceLayerConfig.cluster.vmSize : vmSize - enableAutoScaling: true - minCount: 2 - maxCount: 6 + enableAutoScaling: !enableNodeAutoProvisioning + count: enableNodeAutoProvisioning ? 2 : null + minCount: enableNodeAutoProvisioning ? null : 2 + maxCount: enableNodeAutoProvisioning ? null : 6 securityProfile: { sshAccess: 'Disabled' } @@ -387,9 +395,10 @@ module cluster './managed-cluster/main.bicep' = { name: 'default' mode: 'User' vmSize: empty(vmSize) ? serviceLayerConfig.cluster.defaultSize : vmSize - enableAutoScaling: true - minCount: 4 - maxCount: 20 + enableAutoScaling: !enableNodeAutoProvisioning + count: enableNodeAutoProvisioning ? 4 : null + minCount: enableNodeAutoProvisioning ? null : 4 + maxCount: enableNodeAutoProvisioning ? null : 20 sshAccess: 'Disabled' osType: 'Linux' osSku: 'AzureLinux' @@ -405,9 +414,10 @@ module cluster './managed-cluster/main.bicep' = { name: 'poolz1' mode: 'User' vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize - enableAutoScaling: true - minCount: 1 - maxCount: 3 + enableAutoScaling: !enableNodeAutoProvisioning + minCount: enableNodeAutoProvisioning ? null : 1 + maxCount: enableNodeAutoProvisioning ? null : 3 + count: enableNodeAutoProvisioning ? 1 : null sshAccess: 'Disabled' osType: 'Linux' osSku: 'AzureLinux' @@ -425,9 +435,10 @@ module cluster './managed-cluster/main.bicep' = { name: 'poolz2' mode: 'User' vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize - enableAutoScaling: true - minCount: 1 - maxCount: 3 + enableAutoScaling: !enableNodeAutoProvisioning + minCount: enableNodeAutoProvisioning ? null : 1 + maxCount: enableNodeAutoProvisioning ? null : 3 + count: enableNodeAutoProvisioning ? 1 : null sshAccess: 'Disabled' osType: 'Linux' osSku: 'AzureLinux' @@ -445,9 +456,10 @@ module cluster './managed-cluster/main.bicep' = { name: 'poolz3' mode: 'User' vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize - enableAutoScaling: true - minCount: 1 - maxCount: 3 + enableAutoScaling: !enableNodeAutoProvisioning + minCount: enableNodeAutoProvisioning ? null : 1 + maxCount: enableNodeAutoProvisioning ? null : 3 + count: enableNodeAutoProvisioning ? 1 : null sshAccess: 'Disabled' osType: 'Linux' osSku: 'AzureLinux' diff --git a/docs/src/design_platform.md b/docs/src/design_platform.md index 54e0e50d..9e3caa4a 100644 --- a/docs/src/design_platform.md +++ b/docs/src/design_platform.md @@ -83,7 +83,7 @@ This solution implements comprehensive best practices across security controls a --- - - [x] [Managed Disks](https://learn.microsoft.com/en-us/azure/aks/azure-disk-customer-managed-keys) + - [ ] [Managed Disks](https://learn.microsoft.com/en-us/azure/aks/azure-disk-customer-managed-keys) Secure block-level storage volumes with encryption and access controls. @@ -109,7 +109,7 @@ This solution implements comprehensive best practices across security controls a --- - - [x] [Pod Security Context](https://learn.microsoft.com/en-us/azure/aks/developer-best-practices-pod-security) + - [ ] [Pod Security Context](https://learn.microsoft.com/en-us/azure/aks/developer-best-practices-pod-security) Limit access to processes and services through security context settings, implementing principle of least privilege. diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md index 73439520..913901d4 100644 --- a/docs/src/getting_started.md +++ b/docs/src/getting_started.md @@ -38,6 +38,36 @@ It is recommended to have at least 50 vCPUs in a region for vCPU families along --output json ``` +## Preview Features + +To use AKS Automatic in preview, you must register several feature flags. Register the following features using the [az feature register](https://learn.microsoft.com/en-us/cli/azure/feature?view=azure-cli-latest#az-feature-register) command. + +=== "Command" + ```bash + az feature register --namespace Microsoft.ContainerService --name EnableAPIServerVnetIntegrationPreview + az feature register --namespace Microsoft.ContainerService --name NRGLockdownPreview + az feature register --namespace Microsoft.ContainerService --name SafeguardsPreview + az feature register --namespace Microsoft.ContainerService --name NodeAutoProvisioningPreview + az feature register --namespace Microsoft.ContainerService --name DisableSSHPreview + az feature register --namespace Microsoft.ContainerService --name AutomaticSKUPreview + ``` + +After the features are registered, refresh the registration of the Microsoft.ContainerService resource provider: + +=== "Command" + ```bash + az provider register --namespace Microsoft.ContainerService + ``` + +!!! tip "Verify Registration Status" + Check the registration status using the following command. It may take a few minutes for the status to show *Registered*: + + === "Command" + ```bash + az feature show --namespace Microsoft.ContainerService --name AutomaticSKUPreview + ``` + + ## Resource Providers The following Azure Resource Providers must be registered in your subscription. diff --git a/scripts/post-provision.ps1 b/scripts/post-provision.ps1 index aad92a4d..d007b588 100644 --- a/scripts/post-provision.ps1 +++ b/scripts/post-provision.ps1 @@ -202,7 +202,7 @@ function Update-Application { # Replace double quotes with single quotes in the JSON strings $webUris = $webUris -replace '"', "'" $spaUris = $spaUris -replace '"', "'" - $spaUris += "http://localhost:8080" + # $spaUris += "http://localhost:8080" $jsonPayload = @" { 'web': {'redirectUris': $($webUris),'implicitGrantSettings': {'enableAccessTokenIssuance': false,'enableIdTokenIssuance': false}},