From 76cba621228e2e0b67b3f1ac0001cab86f5c61ee Mon Sep 17 00:00:00 2001 From: Qingchuan Hao Date: Wed, 17 Apr 2024 10:33:31 +0800 Subject: [PATCH] feat: install credential provider support for windows and cache binary in VHD (#4282) --- parts/windows/kuberneteswindowssetup.ps1 | 7 +- pkg/agent/baker_test.go | 44 ++ .../CSECommand | 1 + .../CustomData | 514 ++++++++++++++++++ .../AKSWindows2019+CustomCloud/CustomData | 7 +- .../AKSWindows2019+CustomVnet/CustomData | 7 +- .../CustomData | 7 +- .../testdata/AKSWindows2019+K8S116/CustomData | 7 +- .../testdata/AKSWindows2019+K8S117/CustomData | 7 +- .../testdata/AKSWindows2019+K8S118/CustomData | 7 +- .../AKSWindows2019+K8S119+CSI/CustomData | 7 +- .../AKSWindows2019+K8S119+FIPS/CustomData | 7 +- .../testdata/AKSWindows2019+K8S119/CustomData | 7 +- .../CustomData | 7 +- .../AKSWindows2019+ManagedIdentity/CustomData | 7 +- .../AKSWindows2019+SecurityProfile/CustomData | 7 +- .../CSECommand | 1 + .../CustomData | 508 +++++++++++++++++ pkg/templates/templates_generated.go | 7 +- .../generate-windows-vhd-configuration.ps1 | 3 + vhdbuilder/packer/install-dependencies.sh | 2 +- 21 files changed, 1156 insertions(+), 15 deletions(-) create mode 100644 pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand create mode 100644 pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData create mode 100644 pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand create mode 100644 pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1 index b95092fe2ad..677dd9c8de7 100644 --- a/parts/windows/kuberneteswindowssetup.ps1 +++ b/parts/windows/kuberneteswindowssetup.ps1 @@ -164,6 +164,9 @@ $global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" $global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} $global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} +# Kubelet credential provider +$global:CredentialProviderURL = "{{GetParameter "windowsCredentialProviderURL"}}" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}"); $global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}"; @@ -292,7 +295,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{ AKSCustomCloudContainerRegistryDNSSuffix }}"{{else}}""{{end}} + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 1e967a83d5d..6d6e9fa5cd1 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -1642,6 +1642,50 @@ var _ = Describe("Assert generated customData and cseCmd for Windows", func() { }, } }), + Entry("AKSWindows2019 with out of tree credential provider", "AKSWindows2019+ootcredentialprovider", "1.29.0", func(config *datamodel.NodeBootstrappingConfiguration) { + config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = to.BoolPtr(true) + config.KubeletConfig["--image-credential-provider-config"] = "c:\\var\\lib\\kubelet\\credential-provider-config.yaml" + config.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider" + }), + Entry("AKSWindows2019 with custom cloud and out of tree credential provider", "AKSWindows2019+CustomCloud+ootcredentialprovider", "1.29.0", + func(config *datamodel.NodeBootstrappingConfiguration) { + config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = to.BoolPtr(true) + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{ + Name: "akscustom", + McrURL: "mcr.microsoft.fakecustomcloud", + RepoDepotEndpoint: "https://repodepot.azure.microsoft.fakecustomcloud/ubuntu", + ManagementPortalURL: "https://portal.azure.microsoft.fakecustomcloud/", + PublishSettingsURL: "", + ServiceManagementEndpoint: "https://management.core.microsoft.fakecustomcloud/", + ResourceManagerEndpoint: "https://management.azure.microsoft.fakecustomcloud/", + ActiveDirectoryEndpoint: "https://login.microsoftonline.microsoft.fakecustomcloud/", + GalleryEndpoint: "", + KeyVaultEndpoint: "https://vault.cloudapi.microsoft.fakecustomcloud/", + GraphEndpoint: "https://graph.cloudapi.microsoft.fakecustomcloud/", + ServiceBusEndpoint: "", + BatchManagementEndpoint: "", + StorageEndpointSuffix: "core.microsoft.fakecustomcloud", + SQLDatabaseDNSSuffix: "database.cloudapi.microsoft.fakecustomcloud", + TrafficManagerDNSSuffix: "", + KeyVaultDNSSuffix: "vault.cloudapi.microsoft.fakecustomcloud", + ServiceBusEndpointSuffix: "", + ServiceManagementVMDNSSuffix: "", + ResourceManagerVMDNSSuffix: "cloudapp.azure.microsoft.fakecustomcloud/", + ContainerRegistryDNSSuffix: ".azurecr.microsoft.fakecustomcloud", + CosmosDBDNSSuffix: "documents.core.microsoft.fakecustomcloud/", + TokenAudience: "https://management.core.microsoft.fakecustomcloud/", + ResourceIdentifiers: datamodel.ResourceIdentifiers{ + Graph: "", + KeyVault: "", + Datalake: "", + Batch: "", + OperationalInsights: "", + Storage: "", + }, + } + config.KubeletConfig["--image-credential-provider-config"] = "c:\\var\\lib\\kubelet\\credential-provider-config.yaml" + config.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider" + }), ) }) diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand new file mode 100644 index 00000000000..bee1e387e22 --- /dev/null +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand @@ -0,0 +1 @@ +powershell.exe -ExecutionPolicy Unrestricted -command " $arguments = ' -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''akscustom'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete'; $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; Invoke-Expression('{0} {1}' -f $outputFile, $arguments); " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData new file mode 100644 index 00000000000..5bc6f7d0573 --- /dev/null +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData @@ -0,0 +1,514 @@ +<# + .SYNOPSIS + Provisions VM as a Kubernetes agent. + + .DESCRIPTION + Provisions VM as a Kubernetes agent. + + The parameters passed in are required, and will vary per-deployment. + + Notes on modifying this file: + - This file extension is PS1, but it is actually used as a template from pkg/engine/template_generator.go + - All of the lines that have braces in them will be modified. Please do not change them here, change them in the Go sources + - Single quotes are forbidden, they are reserved to delineate the different members for the ARM template concat() call + - windowscsehelper.ps1 contains basic util functions. It will be compressed to a zip file and then be converted to base64 encoding + string and stored in $zippedFiles. Reason: This script is a template and has some limitations. + - All other scripts will be packaged and published in a single package. It will be downloaded in provisioning VM. + Reason: CustomData has length limitation 87380. + - ProvisioningScriptsPackage contains scripts to start kubelet, kubeproxy, etc. The source is https://github.com/Azure/aks-engine/tree/master/staging/provisioning/windows +#> +[CmdletBinding(DefaultParameterSetName="Standard")] +param( + [string] + [ValidateNotNullOrEmpty()] + $MasterIP, + + [parameter()] + [ValidateNotNullOrEmpty()] + $KubeDnsServiceIp, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $MasterFQDNPrefix, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $Location, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AgentKey, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AADClientId, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AADClientSecret, # base64 + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $NetworkAPIVersion, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $TargetEnvironment, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $LogFile, + + # C:\AzureData\provision.complete + # MUST keep generating this file when CSE is done and do not change the name + # - It is used to avoid running CSE multiple times + # - Some customers use this file to check if CSE is done + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $CSEResultFilePath, + + [string] + $UserAssignedClientID +) +# Do not parse the start time from $LogFile to simplify the logic +$StartTime=Get-Date +$global:ExitCode=0 +$global:ErrorMessage="" +Start-Transcript -Path $LogFile +# These globals will not change between nodes in the same cluster, so they are not +# passed as powershell parameters + +## SSH public keys to add to authorized_keys +$global:SSHKeys = @( "testsshkey" ) + +## Certificates generated by aks-engine +$global:CACertificate = "" +$global:AgentCertificate = "" + +## Download sources provided by aks-engine +$global:KubeBinariesPackageSASURL = "https://acs-mirror.azureedge.net/kubernetes/" +$global:WindowsKubeBinariesURL = "" +$global:KubeBinariesVersion = "1.29.0" +$global:ContainerdUrl = "https://k8swin.blob.core.windows.net/k8s-windows/containerd/containerplat-aks-test-0.0.8.zip" +$global:ContainerdSdnPluginUrl = "" + +## Docker Version +$global:DockerVersion = "20.10.9" + +## ContainerD Usage +$global:DefaultContainerdWindowsSandboxIsolation = "process" +$global:ContainerdWindowsRuntimeHandlers = "" + +## VM configuration passed by Azure +$global:WindowsTelemetryGUID = "fb801154-36b9-41bc-89c2-f4d4f05472b0" + +$global:TenantId = "tenantID" + +$global:SubscriptionId = "subID" +$global:ResourceGroup = "resourceGroupName" +$global:VmType = "vmss" +$global:SubnetName = "aks-subnet" +# NOTE: MasterSubnet is still referenced by `kubeletstart.ps1` and `windowsnodereset.ps1` +# for case of Kubenet +$global:MasterSubnet = "" +$global:SecurityGroupName = "aks-agentpool-36873793-nsg" +$global:VNetName = "aks-vnet-36873793" +$global:RouteTableName = "aks-agentpool-36873793-routetable" +$global:PrimaryAvailabilitySetName = "" +$global:PrimaryScaleSetName = "akswpool2" + +$global:KubeClusterCIDR = "10.240.0.0/16" +$global:KubeServiceCIDR = "10.0.0.0/16" +$global:VNetCIDR = "10.0.0.0/8" + +$global:KubeletNodeLabels = "agentpool=wpool2,kubernetes.azure.com/agentpool=wpool2,kubernetes.azure.com/node-image-version=AKSWindows-2019-17763.1577.201111" + +$global:KubeletConfigArgs = @( "--address=0.0.0.0", "--anonymous-auth=false", "--authentication-token-webhook=true", "--authorization-mode=Webhook", "--azure-container-registry-config=c:\k\azure.json", "--cgroups-per-qos=false", "--client-ca-file=c:\k\ca.crt", "--cloud-config=c:\k\azure.json", "--cloud-provider=azure", "--cluster-dns=10.0.0.10", "--cluster-domain=cluster.local", "--enforce-node-allocatable=", "--event-qps=0", "--eviction-hard=", "--feature-gates=RotateKubeletServerCertificate=true", "--hairpin-mode=promiscuous-bridge", "--image-credential-provider-bin-dir=c:\var\lib\kubelet\credential-provider", "--image-credential-provider-config=c:\var\lib\kubelet\credential-provider-config.yaml", "--image-gc-high-threshold=85", "--image-gc-low-threshold=80", "--keep-terminated-pod-volumes=false", "--kube-reserved=cpu=100m,memory=1843Mi", "--kubeconfig=c:\k\config", "--max-pods=30", "--network-plugin=cni", "--node-status-update-frequency=10s", "--pod-infra-container-image=mcr.microsoft.com/oss/kubernetes/pause:3.9", "--pod-max-pids=-1", "--read-only-port=0", "--resolv-conf=""", "--rotate-certificates=false", "--streaming-connection-idle-timeout=4h", "--system-reserved=memory=2Gi", "--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256" ) +$global:KubeproxyConfigArgs = @( "--metrics-bind-address=0.0.0.0:10249" ) + +$global:KubeproxyFeatureGates = @( "WinDSR=true", "WinOverlay=false" ) + +$global:UseManagedIdentityExtension = "false" +$global:UseInstanceMetadata = "true" + +$global:LoadBalancerSku = "Standard" +$global:ExcludeMasterFromStandardLB = "true" + +$global:PrivateEgressProxyAddress = "" + +# Windows defaults, not changed by aks-engine +$global:CacheDir = "c:\akse-cache" +$global:KubeDir = "c:\k" +$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") + +$global:KubeDnsSearchPath = "svc.cluster.local" + +$global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") +$global:NetworkMode = "L2Bridge" +$global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf") +$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") + + +$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") +$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") +$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") + +# Azure cni configuration +# $global:NetworkPolicy = "" # BUG: unused +$global:NetworkPlugin = "azure" +$global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" +$global:IsDualStackEnabled = $false +$global:IsAzureCNIOverlayEnabled = $false + +# Kubelet credential provider +$global:CredentialProviderURL = "https://acs-mirror.azureedge.net/cloud-provider-azure/v1.29.0/binaries/azure-acr-credential-provider-windows-amd64-v1.29.0.tar.gz" + +# CSI Proxy settings +$global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); +$global:CsiProxyUrl = ""; + +# Hosts Config Agent settings +$global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("false"); + +# These scripts are used by cse +$global:CSEScriptsPackageUrl = ""; + +# The windows nvidia gpu driver related url is used by windows cse +$global:GpuDriverURL = ""; + +# PauseImage +$global:WindowsPauseImageURL = "mcr.microsoft.com/oss/kubernetes/pause:1.4.0"; +$global:AlwaysPullWindowsPauseImage = [System.Convert]::ToBoolean("true"); + +# Calico +$global:WindowsCalicoPackageURL = ""; + +## GPU install +$global:ConfigGPUDriverIfNeeded = [System.Convert]::ToBoolean("true"); + +# GMSA +$global:WindowsGmsaPackageUrl = ""; + +# TLS Bootstrap Token +$global:TLSBootstrapToken = "" + +# Disable OutBoundNAT in Azure CNI configuration +$global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); + +# Base64 representation of ZIP archive +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdR8XVPbyNL/PZ+iS/h/ArWRw0sgG2/pX6vIAnSwZZckQ3ISSiuksa0TWeNnZgRhs/nuT83o3ZaEydnn4nABRuq36e7p6d/MwD44y5AC9Um4ZhBSSCgKgGEI0DyMEdx7NPQhYWEE8yT2WYhjurcPhqC9R4whUqEuSCCMgS1RJpdCEgeIAGXeIowXb3yK3jyGcYAfaX9vbx+GKbcXRYC+hQx8HCAh4jYlAs3WU6WrhDJYecxfwh+fVflf7t0vf+z1FhG+96LBrWEOJ7e2q9m6a880Tbdt5ajxrW5ZE8udmdfm5NZUjmEfLjCBJEbf1shnKABECCbge8liyeD+SYzFF2rvI+x/5bZ9Te4RiRFDNBsKRSxZ99f0uEPlcHJrjibq0L0wRrp7azhXrqU71iflpIPJMG8m17qrf9S1maN+GOnKaQe1kGxOHFf/aNiO8raDVLvStWtXnRqurVs3uuVqE9PUNce4MZxPylkH51Sd2bprjNXLqq7zDo5L3XHt2QdTd9yppV8YH5V3ndSmbqmO7jqTa910LyaWq1pj5dcOFlN3bifWtWuYjm5dqJpuVyx7vwOjOlSnjm5VuI67smesmuqlPtZNxzWmVaau+GvqyNAmwt2Gpruqpk1mplPl7koEbWI6qmHq1lBwGKbtqKORPlSOuxJig8uamaZhXirHXZkxmeqmbV9tqulKiZzlwrD0W3U0ErzaxLwwLmcWZ+7KDsO8UUfG0J2qljrWeRQM01X/NbP0TIRy3JUu5sQdTrRr3XKdifthZoyGWYYWY1eOu3KHJ6emuppuOcaFoamObivHXTlTzONNrpOulNHHU+fTNktXwugmn/CurWvcF87IVk66MuRybKuu/nGqmkNXtbQr40ZXTrqSI2VIlUwnt7plX+mjkTu1jBtjpF/qyklXmghuW3dcS780bMf65E51a2zYtjExlZOubNnmvFFHM+6NziLCuYzxdGI5rqZd6je66djKSWclqbOo1zfT6Wh2aZg5c2dNmTjuxWRmDuuzXTnprCcFU5qJ5mz8QbeU067MKHmm1uTG4P4zzEvX1ixj6tjKaVeK2I5q8RIy1F1L5y5NuVxHta+V065sKbPY1t2pql2rl7py2pUuBcf17INumbqj2yVjV6aUqkyj5OjKkILjyrTd8WQ44+teV3JU5qQosoWWrvQomESeFCxdSVHxmsHD9fFTybdbzSgLcs74tis5eEwdbeoOP5nq2NBckcuWanK+rsRI0y8rjJv18G1XYkxnvAaUa7zytispUj2Oelln6ew8Shd8MEzV+pR3K10ZkftB/6iNZkO95oeuxOB8s2Gz/7pyI+dr0tddNVxtZjuTsTtUHZUPUHn7HvbBQiwhvC8Oad5g4gDxXtKnyF8FvHeExyWKQRt8Uf9MCBp6zPuiJZThFf/Yvw9jCDCiEGPGW2XKnjHDFjVhNnLc0eRSOTv6D6ywdQvRJGL9CC92N0KbTD9x3a42GY14azkxi6p21pW8lm4b/9Ldie0OLb6KnXUue9NZSmbl3YoqNF2oBu9bzjpXwBZexxjrk5mjnHUugC3MN2NXmM/ruj2b8szhdnSuiC2iZlbaStm6o5x1Lo7TmWtfz1zDvJiUC4py1rk6tigtcYpqjGaWrpx1tk9VKWkjZxuXpuoIzq6a2KZf/6jpU/5JOe+qjM/5TP+oK+ddWTabDlWHr7R8PXO10cx2UhjEW87zzpTTszXXmN6c857gvDPJdKda8250S7RI5125lY3I1Sx9qJuOoY7S5mCoW6DA+Rnsl2gZ5phAGFPmRVEYL8AnKEAxC70I1gQ/hAEiO61MXNWwrks5P9/SFODHOMJe0KIK5l4YJQR11YXtQRVuf7elz18i/2ubMh/H83BR6Nzbh2mEPIrACwKI0WO1ym0az1+vPf+rt0g3HKxpSscwLozXObuGA2R6K0RBgd8P9gAApOqosv0G6fX2qyzT0s2GdoK2rYF2jq19gXbS+qZAO13njkA7W+N2QDv51l5AF2nTRkA7fdcuwPNcW1sA7Swt8L/Dtc9A/w7OFti/M0cG+dvpG+H+8+QtUL8zY7thfkeInoX43fm2gbp3mIc7czTC+g7yTUjfYfg2nH+OuB3KP8PZAuNfwpVC+Gc4NuH7zuSb0L0rVxph+y4MVci+C30TXG/n64Lqu2RkCdN3oN6G6DswVeD5DtQlNN+BuA7Ld2CoQvIdyLfg+C48W1C8I3itMLydpwuCt3Ntwu/n5G9A753WhSrsfn7M2xC4m6cZbj/P8xI92zC7m7aGhbt81IpY25k20GpHTetEqi/ny1DqyxkbEerLxVTQaTfzNjJ9ubJNVLqjhA1EuiNXExp9OWsFibYzd6HQDpUbCLSbcht9ttN3IM9damoDiOyYbq1AUNo75GDOnDj6AK6L09YbRGiIK/tgPo6ZF8YgPUgFZhuHcbjyoi2u25AtRx5DlGkpFyIBKCAd909+7R9JsA+3YRSBv/TiBYKQAcPp2S8mBPkMHjLlYn/sEQFN1mtMmMCR+WG1X0rOyPf2YahPLZ03hsMBOEIgR7GCKMOfkJAIHrn2ewQUMQ5H1Ws7h6R9uEXpa4JW+AEJs+Y4ivAjB7IPHgm9+wgVJ+/zhCUE9QuP2Iy/Lkc9zbQqID0c98/7J8ey9yfnOH5zH8YeCRF9U45E3qCRszNv2VsF52/7zCP9xZ/S3r4YWyQc3OSH3JjNENSNedd//5wtFZIWU0r4/oBiRkd4sQjjxTAkXIk2+JJFS2xwfhnhBf0yjZJFGNMv49AnmOI562t4tU4Y6qc7sLa4y6B/Yyjmg/mSCv5SJp3j0a+mtxLDqD91whWymbdap6/2slsX8zBiiGQBpRAnaeoRz2eIUDj4cnQIj8vQX4JHEPjemoc04BGOKV310TcEOGHrhKX5GOHFAgXAlgQniyWs8SMidImiaC9TZAlFZhJFFL5DzwWZoHXk+QhefTl69RpevYIfezkxt5kKm7+D1Du4REweegyBfIHJymOADwfQcyXBkV37gFsSMiSP8OKgt0KUegt0CN/FzO+t6AIUyB/DX6V88T7lnKSj4cR7VbnDbMvmIozQ5AGRK8bWmeCpR7xVuiHDvz6L3xFD5GDsxYHHMHniehlJ0OHdZ8pIGC/uCvLejESvf5p5iCgLY4+bOPXYcmdBYcwqUvRvIdNwgMSTwz3xYx8uQsLn0BL5X3kRoghBOAeP5wyvA2yZ3q/h6RbzbyEFLyLIC57A9/wlCgCnheDmaphGgHNm6fnZmPS5xXeDwSViF9mLA+6NzIAeRR7xl2K361A8Cedw4CDKZM4IeXZrXNcwJHmc67xczzDklROTp1IZPdhkf12a91pw2ULEZM19ezcYqFGUywkRTQ36sVfYlWnsaziJGcgxgqOqQUVagjSjYhsx9VBezfG84h0ZNLx+4lTC1XOCV9DLNXw+ujvk4dgMvVTo4syywdAKMkcVjCBXmLYkiHnlo2xk/DuKKKp71U8ImhLMsI8jWgmNeB3GTQSf7SfK0KpvIta3+fuQPeUUztMa3Q0GKcUQzb0kYq9hFw4anWZ5wr/mmCDPX8JBb53R8RKVy9HjZJWG/saLEkSfN+mwGjvIQ7w9PjnGLFsWKBS6N5kbffeLUjLUqH/sbX+qG0weQh9NcRizsRd7C0S4QzaGwef6hsrSXT0cBVOCFwRROiVojgiKfT4re9tPS6ZGjld2GKGYRU98KQ3jBL2qqMn3uXmh5atePoxh6C1iTFno077N8PrRYz6vBDbzCDPRYyWpGHna8GbPIwuRWd9nJFR4wfgNxogtcaBIl4hJv8EkEXNc2UzwHzU5FmLkSdbwauXFARQfJCN+wF+RbCHKUrESyCpXmSqWOV+IKJylH5+GKPKebOTjOKBwfFRGL7uuV7feRkzOyy2Un3qVZ4RgMs4WKenCC6N0wa0eGvBR90FQiiWwIWXqzhduPqjMGJHQWQlU12sjpuFiyagWhSirXz3eDGymcg/xlgMUMNGjPLn/N+9IpbJdUdfrKPSFw3OJ/aHHPJ4cvKegfdGyOChCK+47qUF4P29geAzzZbeRcErwGhEWIvpZylcQ6Y6ncV5Km7jGPH4+/SwNEyIsHdOUqe4wPfLWFAXjMIpCmka3Lq3NeX2HeP5XMc6DVOVhJTzPTKfmmdm0jOSeQUG6SvCUaF8XftSamWoS7u3cxCjdzUN376G0tjDVfBdPU3dVhmojBsX84EMsfuETFn0LWTkXqtLSoRddeMak1NudfQ6iEsr7QY+mHQ1ae4QbzR948IjJV4/gJA6AhjxKEmACryDAAvvx15ywuLi7RkSIYSIOC1o3omKeclCzttoK/8Vb4f/3TnuV+kIcRZZmV0M5xZStcZzWK4+wDDAQyOLKcYXsYFnkI8gFQJDUa7ufIRDN1vu5nA0xUspSlCNRoXkMfIJ4K+7FwJWzcIWAeVQ0iiQVkbZ+4ywIXmpuWjhs3vskEQq4bDV9I+vfkJ8wBFKJGji+SKtvsuLmS2WZ/kPKLJWLEaftyx9SpnFNwtgP117UpHRavJRnFBEjAPuT7ehj4BmHY94EQLbQqr6f9nRWEo/QA4rgKlwsEWVZ+jASLhYi5iFNfYDzKdvL36UW/BPfO9kDWWUABaI57KtBkC0hB8dnh7xmIl6GQJ4UssQF97DNiyBnfsw9LZdDrLhCzg0oTOMtoYCVnCl3Kvi5U705/y5OunmjyhefalwttAgpQ2TTmDLPsmyQ/TKjjHidsGzxqAxrc9prIsWCnXKsOiVSPrno+n+iwm0VqYskip4HWEpv7kW0Gajl1sxots2wQDEiXgTrhKwxzYpVOt/FN75CyyZmNcyT23HYAi7SsUOQays5xMWDDTPKVZXnUwoZ+HeR/gUpbOiG/5/2BunCsgkSqkvUblakN7hoZaHahyVjazp484Yyz/+KHxCZR/ix7+PVG+/N6duzs/dnZ2/fnL9/d3xyXgaeD8JBqzUmHnkqlWfwf+2RtHXJu9AaAuV8/Les9cwDCL04w6sZ02USBneDgYke+aeMegfvHfwTh3HmyMwSIfqwlrr1nnTnvK0A+xsvCgOPIRMzM4miCdFXa/Z00JSSmZqd9wzaRS89uhSbexXpvGX+G0TXe42s9/4/kVtr5LPJKH7wlD3ohaDA0W/wG1Rn3jY62Ydh2hdEeAECNqRNg/i48p6KvWKKYhqy8AFB4DGvJqMyhYRh0AthAD0/jVe9FSbpXcp/FOGE37mqgqYEBE1opBf+8kvtQYpzQV6gwtlNaJYtCX6Enlt7UYdXolOQ7QihdZt/S6Z6e5ohsLQh4Gm1+0zoaDMLac/V8JqEz3f1nB6FlO0o4Nk2OYxZXXwU4UcU5JTp9snRYZ1jezbwxOykqUNUBY5bs5uHPipD/5tIkJalhtOClcSiJ6i4t3AT9Pv9Mln/0UhTvBaJN1JtR/9oONpkqIu9lTDe9spmPlZNqmhIJwYKIIlp4vuI0nkSRU/pxmV5t6+mU2rNYDn3Xkcq8y8/2wVpmH8b6yTsYHrV8KZpvzWLxI+XbjHo35ZeQnmnRbKEEot0Q7TqfZZKKSJM5shbFyv47tO0a0uby+Mr485F/vm97OZWKtNTa6Ve6rtCysZ99CbUzbv9DHF9SMIoMJPVPSKZ8qyOC0jAm4hsb+MJpKvr0XjwxZ5cOLeqpZdnQ/n5EZjOFy0hvKXIjhilw372QOhpMyI/xsz6o/uKTUqLsYKSPoZM7LVWOKo+lI7fvTs/leB7Pijp+Nej91JlKkgnR6dvf61SRIz6J0cnJ3Wqs9P3NaqT06saxfeeK5YqQfleApmvf/xZhEA6PTp9/076UWHniPzk6OSsKiLbdN5xl67jLx2arm+1bOYxDPMwDgR4yY4OQfgSYuHMQS0Ym5t7rUk19RKK/vuC2kT1wsA2ifhvD6yRXqWXyyNq+YNHUSBPYrm8VCBX410pvC8our1SQ3EU+SJ20/gQxsOQ/BSrhuP5T/FyH/w0Y+1Gxl62Tuy6VZZFpnSbiMsk3hLcunmW/ZkEFCKGcM9lAK7+UX9+ONiHWoQG5clnLXDbF1RKSv7qQ3aPITeOD3kfjPSIdu1R9rpZsDjXBd6DwMwaiX+1kG09N903ya6S9IVw/Zu3WkdoAFKO5z2fyquQz52+uDyBggXqx4hVLlfk/5fhzcNR/6j/9rz5Akb6ruXmBdft4OJqzCqJWLiOUMN9EPoaHhHgOHoSF164K1AcrHEYi9svjQ75O8aW2nhbUb3tw66cSK9bNMcL8T5VtLnSGwkortwSGgi1AMd9yDoEuEdL7yHEJD3iZ9nNnTTYGfVJn9dNJo6tt60MY2B4sYjSP20R926yLS3ABJaYzcNve1mDX/ejHgf0NmTLA+mNVHRh1c2sktoa8TSs82f7vP7mHR7lmQtHgo2bc/A58+bddlE4FFdvPIIcXCH7ibtdh2Ihq10G6DC55VpSwVrxzvV2UmwPg3ttIfYECbClF/OQoP9JvPSo6CeGw/OJdd2wahhd/k9dsqa4joR+ZkQRojQdzn8yBJqCm58ZQsPCyZeb+oNfGuTspcw/vdS8YEnZXja2Zs92jwFyfQwbQ5KLxb6y7oun2TpeXdNBztboYrGuNTh1H/wt0DH3487QsVHC9llkeowrDn8UyYjn4vpZiGMvq0HVw/Syu9y8A1STIvrHPJsqSmh+kK18zg+BJvM5RexuMJgx38SPfQfP4vAbf1M9nM62ont+Wtf5a6Xhzpz09PT0JI/HchDA1dVgtRpQ2p/P51Ll5KEXeZTVrxLWHucH56DA0db465cNuRMkqeaFDeGbVxe3CSvqegdijz9cIbr2YpDT9G9RrXPMUHFH7c5Ww43JXs2EtsuTVYl7myvWllS5llKZH6szWIED5C9x/eFffOo+IMIcLP+T4jgLbXZzkT8BBX5PpVUg2hWmTNyBGIDUQ/HDID9zi/nDclJIo0oIBHE1Jk2EeQhqxMXDCkPWVVTHPNieVT/2pN+bhpS5ovqw2RUp878pjm0xgxvdUVzvFFY3hbNq+WSN0uEYQZW++rhCnTfRA5CO+8dH1Vc1x27kQ5VOL6pBGq/y1wpRxYkVr9SdmDqt4oy/BMzllVscM31vuYb8o/e9XnJ+9LmQvR//GwAA//9QSwcIF/u6NNcUAACSTQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAAF/u6NNcUAACSTQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAACEVAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAACEFwAAAAA=" + +$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" +$fipsEnabled = [System.Convert]::ToBoolean("false") + +# HNS remediator +$global:HNSRemediatorIntervalInMinutes = [System.Convert]::ToUInt32("0"); + +# Log generator +$global:LogGeneratorIntervalInMinutes = [System.Convert]::ToUInt32("0"); + +$global:EnableIncreaseDynamicPortRange = $false + +$global:RebootNeeded = $false + +# Extract cse helper script from ZIP +[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) +Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force + +# Dot-source windowscsehelper.ps1 with functions that are called in this script +. c:\AzureData\windows\windowscsehelper.ps1 +# util functions only can be used after this line, for example, Write-Log + +$global:OperationId = New-Guid + +try +{ + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + + # Exit early if the script has been executed + if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { + Write-Log "The script has been executed before, will exit without doing anything." + return + } + + # This involes using proxy, log the config before fetching packages + Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'" + # TODO update to use proxy + + $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.41.zip" + Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" + Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" + # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl + # but it is better to set the CSE package version in Windows CSE in AgentBaker + # since most changes in CSE package also need the change in Windows CSE in AgentBaker + # In future, AKS RP only sets the endpoint with the pacakge name, for example, https://acs-mirror.azureedge.net/aks/windows/cse/ + if ($global:CSEScriptsPackageUrl.EndsWith("/")) { + $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage + Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" + } + + # Download CSE function scripts + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" + $tempfile = 'c:\csescripts.zip' + DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE + Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" -Force + Remove-Item -Path $tempfile -Force + + # Dot-source cse scripts with functions that are called in this script + . c:\AzureData\windows\azurecnifunc.ps1 + . c:\AzureData\windows\calicofunc.ps1 + . c:\AzureData\windows\configfunc.ps1 + . c:\AzureData\windows\containerdfunc.ps1 + . c:\AzureData\windows\kubeletfunc.ps1 + . c:\AzureData\windows\kubernetesfunc.ps1 + . c:\AzureData\windows\nvidiagpudriverfunc.ps1 + + # Install OpenSSH if SSH enabled + $sshEnabled = [System.Convert]::ToBoolean("true") + + if ( $sshEnabled ) { + Install-OpenSSH -SSHKeys $SSHKeys + } + + Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID + + Resize-OSDrive + + Initialize-DataDisks + + Initialize-DataDirectories + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" + Create-Directory -FullPath "c:\k" + Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" + icacls.exe "c:\k" /inheritance:r + icacls.exe "c:\k" /grant:r SYSTEM:`(OI`)`(CI`)`(F`) + icacls.exe "c:\k" /grant:r BUILTIN\Administrators:`(OI`)`(CI`)`(F`) + icacls.exe "c:\k" /grant:r BUILTIN\Users:`(OI`)`(CI`)`(RX`) + Write-Log "c:\k permissions: " + icacls.exe "c:\k" + Get-ProvisioningScripts + Get-LogCollectionScripts + + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ".azurecr.microsoft.fakecustomcloud" + + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL + + $cniBinPath = $global:AzureCNIBinDir + $cniConfigPath = $global:AzureCNIConfDir + if ($global:NetworkPlugin -eq "kubenet") { + $cniBinPath = $global:CNIPath + $cniConfigPath = $global:CNIConfigPath + } + + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment + + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud + Write-AzureConfig ` + -KubeDir $global:KubeDir ` + -AADClientId $AADClientId ` + -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` + -TenantId $global:TenantId ` + -SubscriptionId $global:SubscriptionId ` + -ResourceGroup $global:ResourceGroup ` + -Location $Location ` + -VmType $global:VmType ` + -SubnetName $global:SubnetName ` + -SecurityGroupName $global:SecurityGroupName ` + -VNetName $global:VNetName ` + -RouteTableName $global:RouteTableName ` + -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName ` + -PrimaryScaleSetName $global:PrimaryScaleSetName ` + -UseManagedIdentityExtension $global:UseManagedIdentityExtension ` + -UserAssignedClientID $UserAssignedClientID ` + -UseInstanceMetadata $global:UseInstanceMetadata ` + -LoadBalancerSku $global:LoadBalancerSku ` + -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB ` + -TargetEnvironment "AzureStackCloud" + + # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud. + # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json + + $azureStackConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") + $envJSON = "eyJuYW1lIjoiQXp1cmVTdGFja0Nsb3VkIiwiTmFtZSI6IkF6dXJlU3RhY2tDbG91ZCIsIm1jclVSTCI6Im1jci5taWNyb3NvZnQuZmFrZWN1c3RvbWNsb3VkIiwicmVwb0RlcG90RW5kcG9pbnQiOiJodHRwczovL3JlcG9kZXBvdC5henVyZS5taWNyb3NvZnQuZmFrZWN1c3RvbWNsb3VkL3VidW50dSIsIm1hbmFnZW1lbnRQb3J0YWxVUkwiOiJodHRwczovL3BvcnRhbC5henVyZS5taWNyb3NvZnQuZmFrZWN1c3RvbWNsb3VkLyIsInNlcnZpY2VNYW5hZ2VtZW50RW5kcG9pbnQiOiJodHRwczovL21hbmFnZW1lbnQuY29yZS5taWNyb3NvZnQuZmFrZWN1c3RvbWNsb3VkLyIsInJlc291cmNlTWFuYWdlckVuZHBvaW50IjoiaHR0cHM6Ly9tYW5hZ2VtZW50LmF6dXJlLm1pY3Jvc29mdC5mYWtlY3VzdG9tY2xvdWQvIiwiYWN0aXZlRGlyZWN0b3J5RW5kcG9pbnQiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5taWNyb3NvZnQuZmFrZWN1c3RvbWNsb3VkLyIsImtleVZhdWx0RW5kcG9pbnQiOiJodHRwczovL3ZhdWx0LmNsb3VkYXBpLm1pY3Jvc29mdC5mYWtlY3VzdG9tY2xvdWQvIiwiZ3JhcGhFbmRwb2ludCI6Imh0dHBzOi8vZ3JhcGguY2xvdWRhcGkubWljcm9zb2Z0LmZha2VjdXN0b21jbG91ZC8iLCJzdG9yYWdlRW5kcG9pbnRTdWZmaXgiOiJjb3JlLm1pY3Jvc29mdC5mYWtlY3VzdG9tY2xvdWQiLCJzcWxEYXRhYmFzZUROU1N1ZmZpeCI6ImRhdGFiYXNlLmNsb3VkYXBpLm1pY3Jvc29mdC5mYWtlY3VzdG9tY2xvdWQiLCJrZXlWYXVsdEROU1N1ZmZpeCI6InZhdWx0LmNsb3VkYXBpLm1pY3Jvc29mdC5mYWtlY3VzdG9tY2xvdWQiLCJyZXNvdXJjZU1hbmFnZXJWTUROU1N1ZmZpeCI6ImNsb3VkYXBwLmF6dXJlLm1pY3Jvc29mdC5mYWtlY3VzdG9tY2xvdWQvIiwiY29udGFpbmVyUmVnaXN0cnlETlNTdWZmaXgiOiIuYXp1cmVjci5taWNyb3NvZnQuZmFrZWN1c3RvbWNsb3VkIiwiY29zbW9zREJETlNTdWZmaXgiOiJkb2N1bWVudHMuY29yZS5taWNyb3NvZnQuZmFrZWN1c3RvbWNsb3VkLyIsInRva2VuQXVkaWVuY2UiOiJodHRwczovL21hbmFnZW1lbnQuY29yZS5taWNyb3NvZnQuZmFrZWN1c3RvbWNsb3VkLyIsInJlc291cmNlSWRlbnRpZmllcnMiOnt9fQ==" + [io.file]::WriteAllBytes($azureStackConfigFile, [System.Convert]::FromBase64String($envJSON)) + + Get-CACertificates + + + Write-CACert -CACertificate $global:CACertificate ` + -KubeDir $global:KubeDir + + if ($global:EnableCsiProxy) { + New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir + } + + if ($global:TLSBootstrapToken) { + Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` + -KubeDir $global:KubeDir ` + -MasterFQDNPrefix $MasterFQDNPrefix ` + -MasterIP $MasterIP ` + -TLSBootstrapToken $global:TLSBootstrapToken + + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled + # This kubeconfig will deleted after calico installation. + # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block + Write-Log "Write temporary kube config" + } else { + Write-Log "Write kube config" + } + + Write-KubeConfig -CACertificate $global:CACertificate ` + -KubeDir $global:KubeDir ` + -MasterFQDNPrefix $MasterFQDNPrefix ` + -MasterIP $MasterIP ` + -AgentKey $AgentKey ` + -AgentCertificate $global:AgentCertificate + + if ($global:EnableHostsConfigAgent) { + New-HostsConfigService + } + + Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin" + + # Configure network policy. + Get-HnsPsm1 -HNSModule $global:HNSModule + Import-Module $global:HNSModule + + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` + -AzureCNIBinDir $global:AzureCNIBinDir ` + -VNetCNIPluginsURL $global:VNetCNIPluginsURL + + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` + -KubeDnsSearchPath $global:KubeDnsSearchPath ` + -KubeClusterCIDR $global:KubeClusterCIDR ` + -KubeServiceCIDR $global:KubeServiceCIDR ` + -VNetCIDR $global:VNetCIDR ` + -IsDualStackEnabled $global:IsDualStackEnabled ` + -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + + if ($TargetEnvironment -ieq "AzureStackCloud") { + GenerateAzureStackCNIConfig ` + -TenantId $global:TenantId ` + -SubscriptionId $global:SubscriptionId ` + -ResourceGroup $global:ResourceGroup ` + -AADClientId $AADClientId ` + -KubeDir $global:KubeDir ` + -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` + -NetworkAPIVersion $NetworkAPIVersion ` + -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) ` + -IdentitySystem "azure_ad" + } + + New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled + + Install-KubernetesServices ` + -KubeDir $global:KubeDir + + Set-Explorer + Adjust-PageFileSize + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" + + Update-ServiceFailureActions + Adjust-DynamicPortRange + Register-LogsCleanupScriptTask + Register-NodeResetScriptTask + Update-DefenderPreferences + + $windowsVersion = Get-WindowsVersion + if ($windowsVersion -ne "1809") { + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" + try { + . C:\k\windowssecuretls.ps1 + Enable-SecureTls + } + catch { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS -ErrorMessage $_ + } + } + + Enable-FIPSMode -FipsEnabled $fipsEnabled + if ($global:WindowsGmsaPackageUrl) { + Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl + } + + Check-APIServerConnectivity -MasterIP $MasterIP + + if ($global:WindowsCalicoPackageURL) { + Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp + } + + Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL + + if (Test-Path $CacheDir) + { + Write-Log "Removing aks cache directory" + Remove-Item $CacheDir -Recurse -Force + } + + if ($global:TLSBootstrapToken) { + Write-Log "Removing temporary kube config" + $kubeConfigFile = [io.path]::Combine($KubeDir, "config") + Remove-Item $kubeConfigFile + } + + Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes + + if ($global:RebootNeeded) { + Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot" + Postpone-RestartComputer + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" + Start-ScheduledTask -TaskName "k8s-restart-job" + + $timeout = 180 ## seconds + $timer = [Diagnostics.Stopwatch]::StartNew() + while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') { + # The task `k8s-restart-job` needs ~8 seconds. + if ($timer.Elapsed.TotalSeconds -gt $timeout) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds" + } + + Write-Log -Message "Waiting on NodeResetScriptTask..." + Start-Sleep -Seconds 3 + } + $timer.Stop() + Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" + } +} +catch +{ + # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block + # Ideally all exceptions will be handled and no exception will be thrown. + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_ +} +finally +{ + # Generate CSE result so it can be returned as the CSE response in csecmd.ps1 + $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date)) + Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode" + # $CSEResultFilePath is used to avoid running CSE multiple times + Set-Content -Path $CSEResultFilePath -Value $global:ExitCode -Force + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Please not use Write-Log or Logs-To-Events after Stop-Transcript + Stop-Transcript + + # Remove the parameters in the log file to avoid leaking secrets + $logs=Get-Content $LogFile | Where-Object {$_ -notmatch "^Host Application: "} + $logs | Set-Content $LogFile + + Upload-GuestVMLogs -ExitCode $global:ExitCode + if ($global:ExitCode -ne 0) { + # $JsonString = "ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" + # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe. + $errorMessageLength = "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length + $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) + throw "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|" + } +} diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData index 19994dd7c42..42ea21ec746 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ".azurecr.microsoft.fakecustomcloud" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData index 20e6c064048..4b1d13f0f77 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData index 6535331c8e9..31492e7e510 100644 --- a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData index de9f95dfac5..e7b6383ebc4 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData index e5db3631775..1d2248f5b57 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData index 2b3729c237a..d6d70e43114 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData index 6239c246d16..bb7f108fe27 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("true"); $global:CsiProxyUrl = "https://acs-mirror.azureedge.net/csi-proxy/v0.1.0/binaries/csi-proxy.tar.gz"; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData index 0c477e882ed..8c6751ad92f 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData index aee83b08dc2..b44a7467851 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData index e01a6b40d94..9439c91c0c7 100644 --- a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData index a3fb5de7dae..332dfa4657c 100644 --- a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData index 02523d5fca8..6789fd4e961 100644 --- a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +# Kubelet credential provider +$global:CredentialProviderURL = "" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); $global:CsiProxyUrl = ""; @@ -288,7 +291,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand new file mode 100644 index 00000000000..dfeda7d5ebb --- /dev/null +++ b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand @@ -0,0 +1 @@ +powershell.exe -ExecutionPolicy Unrestricted -command " $arguments = ' -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete'; $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; Invoke-Expression('{0} {1}' -f $outputFile, $arguments); " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData new file mode 100644 index 00000000000..bddcf4a9eb7 --- /dev/null +++ b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData @@ -0,0 +1,508 @@ +<# + .SYNOPSIS + Provisions VM as a Kubernetes agent. + + .DESCRIPTION + Provisions VM as a Kubernetes agent. + + The parameters passed in are required, and will vary per-deployment. + + Notes on modifying this file: + - This file extension is PS1, but it is actually used as a template from pkg/engine/template_generator.go + - All of the lines that have braces in them will be modified. Please do not change them here, change them in the Go sources + - Single quotes are forbidden, they are reserved to delineate the different members for the ARM template concat() call + - windowscsehelper.ps1 contains basic util functions. It will be compressed to a zip file and then be converted to base64 encoding + string and stored in $zippedFiles. Reason: This script is a template and has some limitations. + - All other scripts will be packaged and published in a single package. It will be downloaded in provisioning VM. + Reason: CustomData has length limitation 87380. + - ProvisioningScriptsPackage contains scripts to start kubelet, kubeproxy, etc. The source is https://github.com/Azure/aks-engine/tree/master/staging/provisioning/windows +#> +[CmdletBinding(DefaultParameterSetName="Standard")] +param( + [string] + [ValidateNotNullOrEmpty()] + $MasterIP, + + [parameter()] + [ValidateNotNullOrEmpty()] + $KubeDnsServiceIp, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $MasterFQDNPrefix, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $Location, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AgentKey, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AADClientId, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AADClientSecret, # base64 + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $NetworkAPIVersion, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $TargetEnvironment, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $LogFile, + + # C:\AzureData\provision.complete + # MUST keep generating this file when CSE is done and do not change the name + # - It is used to avoid running CSE multiple times + # - Some customers use this file to check if CSE is done + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $CSEResultFilePath, + + [string] + $UserAssignedClientID +) +# Do not parse the start time from $LogFile to simplify the logic +$StartTime=Get-Date +$global:ExitCode=0 +$global:ErrorMessage="" +Start-Transcript -Path $LogFile +# These globals will not change between nodes in the same cluster, so they are not +# passed as powershell parameters + +## SSH public keys to add to authorized_keys +$global:SSHKeys = @( "testsshkey" ) + +## Certificates generated by aks-engine +$global:CACertificate = "" +$global:AgentCertificate = "" + +## Download sources provided by aks-engine +$global:KubeBinariesPackageSASURL = "https://acs-mirror.azureedge.net/kubernetes/" +$global:WindowsKubeBinariesURL = "" +$global:KubeBinariesVersion = "1.29.0" +$global:ContainerdUrl = "https://k8swin.blob.core.windows.net/k8s-windows/containerd/containerplat-aks-test-0.0.8.zip" +$global:ContainerdSdnPluginUrl = "" + +## Docker Version +$global:DockerVersion = "20.10.9" + +## ContainerD Usage +$global:DefaultContainerdWindowsSandboxIsolation = "process" +$global:ContainerdWindowsRuntimeHandlers = "" + +## VM configuration passed by Azure +$global:WindowsTelemetryGUID = "fb801154-36b9-41bc-89c2-f4d4f05472b0" + +$global:TenantId = "tenantID" + +$global:SubscriptionId = "subID" +$global:ResourceGroup = "resourceGroupName" +$global:VmType = "vmss" +$global:SubnetName = "aks-subnet" +# NOTE: MasterSubnet is still referenced by `kubeletstart.ps1` and `windowsnodereset.ps1` +# for case of Kubenet +$global:MasterSubnet = "" +$global:SecurityGroupName = "aks-agentpool-36873793-nsg" +$global:VNetName = "aks-vnet-36873793" +$global:RouteTableName = "aks-agentpool-36873793-routetable" +$global:PrimaryAvailabilitySetName = "" +$global:PrimaryScaleSetName = "akswpool2" + +$global:KubeClusterCIDR = "10.240.0.0/16" +$global:KubeServiceCIDR = "10.0.0.0/16" +$global:VNetCIDR = "10.0.0.0/8" + +$global:KubeletNodeLabels = "agentpool=wpool2,kubernetes.azure.com/agentpool=wpool2,kubernetes.azure.com/node-image-version=AKSWindows-2019-17763.1577.201111" + +$global:KubeletConfigArgs = @( "--address=0.0.0.0", "--anonymous-auth=false", "--authentication-token-webhook=true", "--authorization-mode=Webhook", "--azure-container-registry-config=c:\k\azure.json", "--cgroups-per-qos=false", "--client-ca-file=c:\k\ca.crt", "--cloud-config=c:\k\azure.json", "--cloud-provider=azure", "--cluster-dns=10.0.0.10", "--cluster-domain=cluster.local", "--enforce-node-allocatable=", "--event-qps=0", "--eviction-hard=", "--feature-gates=RotateKubeletServerCertificate=true", "--hairpin-mode=promiscuous-bridge", "--image-credential-provider-bin-dir=c:\var\lib\kubelet\credential-provider", "--image-credential-provider-config=c:\var\lib\kubelet\credential-provider-config.yaml", "--image-gc-high-threshold=85", "--image-gc-low-threshold=80", "--keep-terminated-pod-volumes=false", "--kube-reserved=cpu=100m,memory=1843Mi", "--kubeconfig=c:\k\config", "--max-pods=30", "--network-plugin=cni", "--node-status-update-frequency=10s", "--pod-infra-container-image=mcr.microsoft.com/oss/kubernetes/pause:3.9", "--pod-max-pids=-1", "--read-only-port=0", "--resolv-conf=""", "--rotate-certificates=false", "--streaming-connection-idle-timeout=4h", "--system-reserved=memory=2Gi", "--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256" ) +$global:KubeproxyConfigArgs = @( "--metrics-bind-address=0.0.0.0:10249" ) + +$global:KubeproxyFeatureGates = @( "WinDSR=true", "WinOverlay=false" ) + +$global:UseManagedIdentityExtension = "false" +$global:UseInstanceMetadata = "true" + +$global:LoadBalancerSku = "Standard" +$global:ExcludeMasterFromStandardLB = "true" + +$global:PrivateEgressProxyAddress = "" + +# Windows defaults, not changed by aks-engine +$global:CacheDir = "c:\akse-cache" +$global:KubeDir = "c:\k" +$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") + +$global:KubeDnsSearchPath = "svc.cluster.local" + +$global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") +$global:NetworkMode = "L2Bridge" +$global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf") +$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") + + +$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") +$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") +$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") + +# Azure cni configuration +# $global:NetworkPolicy = "" # BUG: unused +$global:NetworkPlugin = "azure" +$global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" +$global:IsDualStackEnabled = $false +$global:IsAzureCNIOverlayEnabled = $false + +# Kubelet credential provider +$global:CredentialProviderURL = "https://acs-mirror.azureedge.net/cloud-provider-azure/v1.29.0/binaries/azure-acr-credential-provider-windows-amd64-v1.29.0.tar.gz" + +# CSI Proxy settings +$global:EnableCsiProxy = [System.Convert]::ToBoolean("false"); +$global:CsiProxyUrl = ""; + +# Hosts Config Agent settings +$global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("false"); + +# These scripts are used by cse +$global:CSEScriptsPackageUrl = ""; + +# The windows nvidia gpu driver related url is used by windows cse +$global:GpuDriverURL = ""; + +# PauseImage +$global:WindowsPauseImageURL = "mcr.microsoft.com/oss/kubernetes/pause:1.4.0"; +$global:AlwaysPullWindowsPauseImage = [System.Convert]::ToBoolean("true"); + +# Calico +$global:WindowsCalicoPackageURL = ""; + +## GPU install +$global:ConfigGPUDriverIfNeeded = [System.Convert]::ToBoolean("true"); + +# GMSA +$global:WindowsGmsaPackageUrl = ""; + +# TLS Bootstrap Token +$global:TLSBootstrapToken = "" + +# Disable OutBoundNAT in Azure CNI configuration +$global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); + +# Base64 representation of ZIP archive +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdR8XVPbyNL/PZ+iS/h/ArWRw0sgG2/pX6vIAnSwZZckQ3ISSiuksa0TWeNnZgRhs/nuT83o3ZaEydnn4nABRuq36e7p6d/MwD44y5AC9Um4ZhBSSCgKgGEI0DyMEdx7NPQhYWEE8yT2WYhjurcPhqC9R4whUqEuSCCMgS1RJpdCEgeIAGXeIowXb3yK3jyGcYAfaX9vbx+GKbcXRYC+hQx8HCAh4jYlAs3WU6WrhDJYecxfwh+fVflf7t0vf+z1FhG+96LBrWEOJ7e2q9m6a880Tbdt5ajxrW5ZE8udmdfm5NZUjmEfLjCBJEbf1shnKABECCbge8liyeD+SYzFF2rvI+x/5bZ9Te4RiRFDNBsKRSxZ99f0uEPlcHJrjibq0L0wRrp7azhXrqU71iflpIPJMG8m17qrf9S1maN+GOnKaQe1kGxOHFf/aNiO8raDVLvStWtXnRqurVs3uuVqE9PUNce4MZxPylkH51Sd2bprjNXLqq7zDo5L3XHt2QdTd9yppV8YH5V3ndSmbqmO7jqTa910LyaWq1pj5dcOFlN3bifWtWuYjm5dqJpuVyx7vwOjOlSnjm5VuI67smesmuqlPtZNxzWmVaau+GvqyNAmwt2Gpruqpk1mplPl7koEbWI6qmHq1lBwGKbtqKORPlSOuxJig8uamaZhXirHXZkxmeqmbV9tqulKiZzlwrD0W3U0ErzaxLwwLmcWZ+7KDsO8UUfG0J2qljrWeRQM01X/NbP0TIRy3JUu5sQdTrRr3XKdifthZoyGWYYWY1eOu3KHJ6emuppuOcaFoamObivHXTlTzONNrpOulNHHU+fTNktXwugmn/CurWvcF87IVk66MuRybKuu/nGqmkNXtbQr40ZXTrqSI2VIlUwnt7plX+mjkTu1jBtjpF/qyklXmghuW3dcS780bMf65E51a2zYtjExlZOubNnmvFFHM+6NziLCuYzxdGI5rqZd6je66djKSWclqbOo1zfT6Wh2aZg5c2dNmTjuxWRmDuuzXTnprCcFU5qJ5mz8QbeU067MKHmm1uTG4P4zzEvX1ixj6tjKaVeK2I5q8RIy1F1L5y5NuVxHta+V065sKbPY1t2pql2rl7py2pUuBcf17INumbqj2yVjV6aUqkyj5OjKkILjyrTd8WQ44+teV3JU5qQosoWWrvQomESeFCxdSVHxmsHD9fFTybdbzSgLcs74tis5eEwdbeoOP5nq2NBckcuWanK+rsRI0y8rjJv18G1XYkxnvAaUa7zytispUj2Oelln6ew8Shd8MEzV+pR3K10ZkftB/6iNZkO95oeuxOB8s2Gz/7pyI+dr0tddNVxtZjuTsTtUHZUPUHn7HvbBQiwhvC8Oad5g4gDxXtKnyF8FvHeExyWKQRt8Uf9MCBp6zPuiJZThFf/Yvw9jCDCiEGPGW2XKnjHDFjVhNnLc0eRSOTv6D6ywdQvRJGL9CC92N0KbTD9x3a42GY14azkxi6p21pW8lm4b/9Ldie0OLb6KnXUue9NZSmbl3YoqNF2oBu9bzjpXwBZexxjrk5mjnHUugC3MN2NXmM/ruj2b8szhdnSuiC2iZlbaStm6o5x1Lo7TmWtfz1zDvJiUC4py1rk6tigtcYpqjGaWrpx1tk9VKWkjZxuXpuoIzq6a2KZf/6jpU/5JOe+qjM/5TP+oK+ddWTabDlWHr7R8PXO10cx2UhjEW87zzpTTszXXmN6c857gvDPJdKda8250S7RI5125lY3I1Sx9qJuOoY7S5mCoW6DA+Rnsl2gZ5phAGFPmRVEYL8AnKEAxC70I1gQ/hAEiO61MXNWwrks5P9/SFODHOMJe0KIK5l4YJQR11YXtQRVuf7elz18i/2ubMh/H83BR6Nzbh2mEPIrACwKI0WO1ym0az1+vPf+rt0g3HKxpSscwLozXObuGA2R6K0RBgd8P9gAApOqosv0G6fX2qyzT0s2GdoK2rYF2jq19gXbS+qZAO13njkA7W+N2QDv51l5AF2nTRkA7fdcuwPNcW1sA7Swt8L/Dtc9A/w7OFti/M0cG+dvpG+H+8+QtUL8zY7thfkeInoX43fm2gbp3mIc7czTC+g7yTUjfYfg2nH+OuB3KP8PZAuNfwpVC+Gc4NuH7zuSb0L0rVxph+y4MVci+C30TXG/n64Lqu2RkCdN3oN6G6DswVeD5DtQlNN+BuA7Ld2CoQvIdyLfg+C48W1C8I3itMLydpwuCt3Ntwu/n5G9A753WhSrsfn7M2xC4m6cZbj/P8xI92zC7m7aGhbt81IpY25k20GpHTetEqi/ny1DqyxkbEerLxVTQaTfzNjJ9ubJNVLqjhA1EuiNXExp9OWsFibYzd6HQDpUbCLSbcht9ttN3IM9damoDiOyYbq1AUNo75GDOnDj6AK6L09YbRGiIK/tgPo6ZF8YgPUgFZhuHcbjyoi2u25AtRx5DlGkpFyIBKCAd909+7R9JsA+3YRSBv/TiBYKQAcPp2S8mBPkMHjLlYn/sEQFN1mtMmMCR+WG1X0rOyPf2YahPLZ03hsMBOEIgR7GCKMOfkJAIHrn2ewQUMQ5H1Ws7h6R9uEXpa4JW+AEJs+Y4ivAjB7IPHgm9+wgVJ+/zhCUE9QuP2Iy/Lkc9zbQqID0c98/7J8ey9yfnOH5zH8YeCRF9U45E3qCRszNv2VsF52/7zCP9xZ/S3r4YWyQc3OSH3JjNENSNedd//5wtFZIWU0r4/oBiRkd4sQjjxTAkXIk2+JJFS2xwfhnhBf0yjZJFGNMv49AnmOI562t4tU4Y6qc7sLa4y6B/Yyjmg/mSCv5SJp3j0a+mtxLDqD91whWymbdap6/2slsX8zBiiGQBpRAnaeoRz2eIUDj4cnQIj8vQX4JHEPjemoc04BGOKV310TcEOGHrhKX5GOHFAgXAlgQniyWs8SMidImiaC9TZAlFZhJFFL5DzwWZoHXk+QhefTl69RpevYIfezkxt5kKm7+D1Du4REweegyBfIHJymOADwfQcyXBkV37gFsSMiSP8OKgt0KUegt0CN/FzO+t6AIUyB/DX6V88T7lnKSj4cR7VbnDbMvmIozQ5AGRK8bWmeCpR7xVuiHDvz6L3xFD5GDsxYHHMHniehlJ0OHdZ8pIGC/uCvLejESvf5p5iCgLY4+bOPXYcmdBYcwqUvRvIdNwgMSTwz3xYx8uQsLn0BL5X3kRoghBOAeP5wyvA2yZ3q/h6RbzbyEFLyLIC57A9/wlCgCnheDmaphGgHNm6fnZmPS5xXeDwSViF9mLA+6NzIAeRR7xl2K361A8Cedw4CDKZM4IeXZrXNcwJHmc67xczzDklROTp1IZPdhkf12a91pw2ULEZM19ezcYqFGUywkRTQ36sVfYlWnsaziJGcgxgqOqQUVagjSjYhsx9VBezfG84h0ZNLx+4lTC1XOCV9DLNXw+ujvk4dgMvVTo4syywdAKMkcVjCBXmLYkiHnlo2xk/DuKKKp71U8ImhLMsI8jWgmNeB3GTQSf7SfK0KpvIta3+fuQPeUUztMa3Q0GKcUQzb0kYq9hFw4anWZ5wr/mmCDPX8JBb53R8RKVy9HjZJWG/saLEkSfN+mwGjvIQ7w9PjnGLFsWKBS6N5kbffeLUjLUqH/sbX+qG0weQh9NcRizsRd7C0S4QzaGwef6hsrSXT0cBVOCFwRROiVojgiKfT4re9tPS6ZGjld2GKGYRU98KQ3jBL2qqMn3uXmh5atePoxh6C1iTFno077N8PrRYz6vBDbzCDPRYyWpGHna8GbPIwuRWd9nJFR4wfgNxogtcaBIl4hJv8EkEXNc2UzwHzU5FmLkSdbwauXFARQfJCN+wF+RbCHKUrESyCpXmSqWOV+IKJylH5+GKPKebOTjOKBwfFRGL7uuV7feRkzOyy2Un3qVZ4RgMs4WKenCC6N0wa0eGvBR90FQiiWwIWXqzhduPqjMGJHQWQlU12sjpuFiyagWhSirXz3eDGymcg/xlgMUMNGjPLn/N+9IpbJdUdfrKPSFw3OJ/aHHPJ4cvKegfdGyOChCK+47qUF4P29geAzzZbeRcErwGhEWIvpZylcQ6Y6ncV5Km7jGPH4+/SwNEyIsHdOUqe4wPfLWFAXjMIpCmka3Lq3NeX2HeP5XMc6DVOVhJTzPTKfmmdm0jOSeQUG6SvCUaF8XftSamWoS7u3cxCjdzUN376G0tjDVfBdPU3dVhmojBsX84EMsfuETFn0LWTkXqtLSoRddeMak1NudfQ6iEsr7QY+mHQ1ae4QbzR948IjJV4/gJA6AhjxKEmACryDAAvvx15ywuLi7RkSIYSIOC1o3omKeclCzttoK/8Vb4f/3TnuV+kIcRZZmV0M5xZStcZzWK4+wDDAQyOLKcYXsYFnkI8gFQJDUa7ufIRDN1vu5nA0xUspSlCNRoXkMfIJ4K+7FwJWzcIWAeVQ0iiQVkbZ+4ywIXmpuWjhs3vskEQq4bDV9I+vfkJ8wBFKJGji+SKtvsuLmS2WZ/kPKLJWLEaftyx9SpnFNwtgP117UpHRavJRnFBEjAPuT7ehj4BmHY94EQLbQqr6f9nRWEo/QA4rgKlwsEWVZ+jASLhYi5iFNfYDzKdvL36UW/BPfO9kDWWUABaI57KtBkC0hB8dnh7xmIl6GQJ4UssQF97DNiyBnfsw9LZdDrLhCzg0oTOMtoYCVnCl3Kvi5U705/y5OunmjyhefalwttAgpQ2TTmDLPsmyQ/TKjjHidsGzxqAxrc9prIsWCnXKsOiVSPrno+n+iwm0VqYskip4HWEpv7kW0Gajl1sxots2wQDEiXgTrhKwxzYpVOt/FN75CyyZmNcyT23HYAi7SsUOQays5xMWDDTPKVZXnUwoZ+HeR/gUpbOiG/5/2BunCsgkSqkvUblakN7hoZaHahyVjazp484Yyz/+KHxCZR/ix7+PVG+/N6duzs/dnZ2/fnL9/d3xyXgaeD8JBqzUmHnkqlWfwf+2RtHXJu9AaAuV8/Les9cwDCL04w6sZ02USBneDgYke+aeMegfvHfwTh3HmyMwSIfqwlrr1nnTnvK0A+xsvCgOPIRMzM4miCdFXa/Z00JSSmZqd9wzaRS89uhSbexXpvGX+G0TXe42s9/4/kVtr5LPJKH7wlD3ohaDA0W/wG1Rn3jY62Ydh2hdEeAECNqRNg/i48p6KvWKKYhqy8AFB4DGvJqMyhYRh0AthAD0/jVe9FSbpXcp/FOGE37mqgqYEBE1opBf+8kvtQYpzQV6gwtlNaJYtCX6Enlt7UYdXolOQ7QihdZt/S6Z6e5ohsLQh4Gm1+0zoaDMLac/V8JqEz3f1nB6FlO0o4Nk2OYxZXXwU4UcU5JTp9snRYZ1jezbwxOykqUNUBY5bs5uHPipD/5tIkJalhtOClcSiJ6i4t3AT9Pv9Mln/0UhTvBaJN1JtR/9oONpkqIu9lTDe9spmPlZNqmhIJwYKIIlp4vuI0nkSRU/pxmV5t6+mU2rNYDn3Xkcq8y8/2wVpmH8b6yTsYHrV8KZpvzWLxI+XbjHo35ZeQnmnRbKEEot0Q7TqfZZKKSJM5shbFyv47tO0a0uby+Mr485F/vm97OZWKtNTa6Ve6rtCysZ99CbUzbv9DHF9SMIoMJPVPSKZ8qyOC0jAm4hsb+MJpKvr0XjwxZ5cOLeqpZdnQ/n5EZjOFy0hvKXIjhilw372QOhpMyI/xsz6o/uKTUqLsYKSPoZM7LVWOKo+lI7fvTs/leB7Pijp+Nej91JlKkgnR6dvf61SRIz6J0cnJ3Wqs9P3NaqT06saxfeeK5YqQfleApmvf/xZhEA6PTp9/076UWHniPzk6OSsKiLbdN5xl67jLx2arm+1bOYxDPMwDgR4yY4OQfgSYuHMQS0Ym5t7rUk19RKK/vuC2kT1wsA2ifhvD6yRXqWXyyNq+YNHUSBPYrm8VCBX410pvC8our1SQ3EU+SJ20/gQxsOQ/BSrhuP5T/FyH/w0Y+1Gxl62Tuy6VZZFpnSbiMsk3hLcunmW/ZkEFCKGcM9lAK7+UX9+ONiHWoQG5clnLXDbF1RKSv7qQ3aPITeOD3kfjPSIdu1R9rpZsDjXBd6DwMwaiX+1kG09N903ya6S9IVw/Zu3WkdoAFKO5z2fyquQz52+uDyBggXqx4hVLlfk/5fhzcNR/6j/9rz5Akb6ruXmBdft4OJqzCqJWLiOUMN9EPoaHhHgOHoSF164K1AcrHEYi9svjQ75O8aW2nhbUb3tw66cSK9bNMcL8T5VtLnSGwkortwSGgi1AMd9yDoEuEdL7yHEJD3iZ9nNnTTYGfVJn9dNJo6tt60MY2B4sYjSP20R926yLS3ABJaYzcNve1mDX/ejHgf0NmTLA+mNVHRh1c2sktoa8TSs82f7vP7mHR7lmQtHgo2bc/A58+bddlE4FFdvPIIcXCH7ibtdh2Ihq10G6DC55VpSwVrxzvV2UmwPg3ttIfYECbClF/OQoP9JvPSo6CeGw/OJdd2wahhd/k9dsqa4joR+ZkQRojQdzn8yBJqCm58ZQsPCyZeb+oNfGuTspcw/vdS8YEnZXja2Zs92jwFyfQwbQ5KLxb6y7oun2TpeXdNBztboYrGuNTh1H/wt0DH3487QsVHC9llkeowrDn8UyYjn4vpZiGMvq0HVw/Syu9y8A1STIvrHPJsqSmh+kK18zg+BJvM5RexuMJgx38SPfQfP4vAbf1M9nM62ont+Wtf5a6Xhzpz09PT0JI/HchDA1dVgtRpQ2p/P51Ll5KEXeZTVrxLWHucH56DA0db465cNuRMkqeaFDeGbVxe3CSvqegdijz9cIbr2YpDT9G9RrXPMUHFH7c5Ww43JXs2EtsuTVYl7myvWllS5llKZH6szWIED5C9x/eFffOo+IMIcLP+T4jgLbXZzkT8BBX5PpVUg2hWmTNyBGIDUQ/HDID9zi/nDclJIo0oIBHE1Jk2EeQhqxMXDCkPWVVTHPNieVT/2pN+bhpS5ovqw2RUp878pjm0xgxvdUVzvFFY3hbNq+WSN0uEYQZW++rhCnTfRA5CO+8dH1Vc1x27kQ5VOL6pBGq/y1wpRxYkVr9SdmDqt4oy/BMzllVscM31vuYb8o/e9XnJ+9LmQvR//GwAA//9QSwcIF/u6NNcUAACSTQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAAF/u6NNcUAACSTQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAACEVAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAACEFwAAAAA=" + +$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" +$fipsEnabled = [System.Convert]::ToBoolean("false") + +# HNS remediator +$global:HNSRemediatorIntervalInMinutes = [System.Convert]::ToUInt32("0"); + +# Log generator +$global:LogGeneratorIntervalInMinutes = [System.Convert]::ToUInt32("0"); + +$global:EnableIncreaseDynamicPortRange = $false + +$global:RebootNeeded = $false + +# Extract cse helper script from ZIP +[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) +Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force + +# Dot-source windowscsehelper.ps1 with functions that are called in this script +. c:\AzureData\windows\windowscsehelper.ps1 +# util functions only can be used after this line, for example, Write-Log + +$global:OperationId = New-Guid + +try +{ + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + + # Exit early if the script has been executed + if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { + Write-Log "The script has been executed before, will exit without doing anything." + return + } + + # This involes using proxy, log the config before fetching packages + Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'" + # TODO update to use proxy + + $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.41.zip" + Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" + Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" + # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl + # but it is better to set the CSE package version in Windows CSE in AgentBaker + # since most changes in CSE package also need the change in Windows CSE in AgentBaker + # In future, AKS RP only sets the endpoint with the pacakge name, for example, https://acs-mirror.azureedge.net/aks/windows/cse/ + if ($global:CSEScriptsPackageUrl.EndsWith("/")) { + $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage + Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" + } + + # Download CSE function scripts + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" + $tempfile = 'c:\csescripts.zip' + DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE + Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" -Force + Remove-Item -Path $tempfile -Force + + # Dot-source cse scripts with functions that are called in this script + . c:\AzureData\windows\azurecnifunc.ps1 + . c:\AzureData\windows\calicofunc.ps1 + . c:\AzureData\windows\configfunc.ps1 + . c:\AzureData\windows\containerdfunc.ps1 + . c:\AzureData\windows\kubeletfunc.ps1 + . c:\AzureData\windows\kubernetesfunc.ps1 + . c:\AzureData\windows\nvidiagpudriverfunc.ps1 + + # Install OpenSSH if SSH enabled + $sshEnabled = [System.Convert]::ToBoolean("true") + + if ( $sshEnabled ) { + Install-OpenSSH -SSHKeys $SSHKeys + } + + Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID + + Resize-OSDrive + + Initialize-DataDisks + + Initialize-DataDirectories + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" + Create-Directory -FullPath "c:\k" + Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" + icacls.exe "c:\k" /inheritance:r + icacls.exe "c:\k" /grant:r SYSTEM:`(OI`)`(CI`)`(F`) + icacls.exe "c:\k" /grant:r BUILTIN\Administrators:`(OI`)`(CI`)`(F`) + icacls.exe "c:\k" /grant:r BUILTIN\Users:`(OI`)`(CI`)`(RX`) + Write-Log "c:\k permissions: " + icacls.exe "c:\k" + Get-ProvisioningScripts + Get-LogCollectionScripts + + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix "" + + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL + + $cniBinPath = $global:AzureCNIBinDir + $cniConfigPath = $global:AzureCNIConfDir + if ($global:NetworkPlugin -eq "kubenet") { + $cniBinPath = $global:CNIPath + $cniConfigPath = $global:CNIConfigPath + } + + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment + + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud + Write-AzureConfig ` + -KubeDir $global:KubeDir ` + -AADClientId $AADClientId ` + -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` + -TenantId $global:TenantId ` + -SubscriptionId $global:SubscriptionId ` + -ResourceGroup $global:ResourceGroup ` + -Location $Location ` + -VmType $global:VmType ` + -SubnetName $global:SubnetName ` + -SecurityGroupName $global:SecurityGroupName ` + -VNetName $global:VNetName ` + -RouteTableName $global:RouteTableName ` + -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName ` + -PrimaryScaleSetName $global:PrimaryScaleSetName ` + -UseManagedIdentityExtension $global:UseManagedIdentityExtension ` + -UserAssignedClientID $UserAssignedClientID ` + -UseInstanceMetadata $global:UseInstanceMetadata ` + -LoadBalancerSku $global:LoadBalancerSku ` + -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB ` + -TargetEnvironment $TargetEnvironment + + # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud. + # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json + + + Write-CACert -CACertificate $global:CACertificate ` + -KubeDir $global:KubeDir + + if ($global:EnableCsiProxy) { + New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir + } + + if ($global:TLSBootstrapToken) { + Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` + -KubeDir $global:KubeDir ` + -MasterFQDNPrefix $MasterFQDNPrefix ` + -MasterIP $MasterIP ` + -TLSBootstrapToken $global:TLSBootstrapToken + + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled + # This kubeconfig will deleted after calico installation. + # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block + Write-Log "Write temporary kube config" + } else { + Write-Log "Write kube config" + } + + Write-KubeConfig -CACertificate $global:CACertificate ` + -KubeDir $global:KubeDir ` + -MasterFQDNPrefix $MasterFQDNPrefix ` + -MasterIP $MasterIP ` + -AgentKey $AgentKey ` + -AgentCertificate $global:AgentCertificate + + if ($global:EnableHostsConfigAgent) { + New-HostsConfigService + } + + Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin" + + # Configure network policy. + Get-HnsPsm1 -HNSModule $global:HNSModule + Import-Module $global:HNSModule + + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` + -AzureCNIBinDir $global:AzureCNIBinDir ` + -VNetCNIPluginsURL $global:VNetCNIPluginsURL + + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` + -KubeDnsSearchPath $global:KubeDnsSearchPath ` + -KubeClusterCIDR $global:KubeClusterCIDR ` + -KubeServiceCIDR $global:KubeServiceCIDR ` + -VNetCIDR $global:VNetCIDR ` + -IsDualStackEnabled $global:IsDualStackEnabled ` + -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + + if ($TargetEnvironment -ieq "AzureStackCloud") { + GenerateAzureStackCNIConfig ` + -TenantId $global:TenantId ` + -SubscriptionId $global:SubscriptionId ` + -ResourceGroup $global:ResourceGroup ` + -AADClientId $AADClientId ` + -KubeDir $global:KubeDir ` + -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` + -NetworkAPIVersion $NetworkAPIVersion ` + -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) ` + -IdentitySystem "azure_ad" + } + + New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled + + Install-KubernetesServices ` + -KubeDir $global:KubeDir + + Set-Explorer + Adjust-PageFileSize + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" + + Update-ServiceFailureActions + Adjust-DynamicPortRange + Register-LogsCleanupScriptTask + Register-NodeResetScriptTask + Update-DefenderPreferences + + $windowsVersion = Get-WindowsVersion + if ($windowsVersion -ne "1809") { + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" + try { + . C:\k\windowssecuretls.ps1 + Enable-SecureTls + } + catch { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS -ErrorMessage $_ + } + } + + Enable-FIPSMode -FipsEnabled $fipsEnabled + if ($global:WindowsGmsaPackageUrl) { + Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl + } + + Check-APIServerConnectivity -MasterIP $MasterIP + + if ($global:WindowsCalicoPackageURL) { + Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp + } + + Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL + + if (Test-Path $CacheDir) + { + Write-Log "Removing aks cache directory" + Remove-Item $CacheDir -Recurse -Force + } + + if ($global:TLSBootstrapToken) { + Write-Log "Removing temporary kube config" + $kubeConfigFile = [io.path]::Combine($KubeDir, "config") + Remove-Item $kubeConfigFile + } + + Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes + + if ($global:RebootNeeded) { + Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot" + Postpone-RestartComputer + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" + Start-ScheduledTask -TaskName "k8s-restart-job" + + $timeout = 180 ## seconds + $timer = [Diagnostics.Stopwatch]::StartNew() + while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') { + # The task `k8s-restart-job` needs ~8 seconds. + if ($timer.Elapsed.TotalSeconds -gt $timeout) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds" + } + + Write-Log -Message "Waiting on NodeResetScriptTask..." + Start-Sleep -Seconds 3 + } + $timer.Stop() + Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" + } +} +catch +{ + # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block + # Ideally all exceptions will be handled and no exception will be thrown. + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_ +} +finally +{ + # Generate CSE result so it can be returned as the CSE response in csecmd.ps1 + $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date)) + Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode" + # $CSEResultFilePath is used to avoid running CSE multiple times + Set-Content -Path $CSEResultFilePath -Value $global:ExitCode -Force + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Please not use Write-Log or Logs-To-Events after Stop-Transcript + Stop-Transcript + + # Remove the parameters in the log file to avoid leaking secrets + $logs=Get-Content $LogFile | Where-Object {$_ -notmatch "^Host Application: "} + $logs | Set-Content $LogFile + + Upload-GuestVMLogs -ExitCode $global:ExitCode + if ($global:ExitCode -ne 0) { + # $JsonString = "ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" + # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe. + $errorMessageLength = "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length + $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) + throw "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|" + } +} diff --git a/pkg/templates/templates_generated.go b/pkg/templates/templates_generated.go index 6c127341c2f..24cb68792ce 100644 --- a/pkg/templates/templates_generated.go +++ b/pkg/templates/templates_generated.go @@ -8568,6 +8568,9 @@ $global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" $global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} $global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} +# Kubelet credential provider +$global:CredentialProviderURL = "{{GetParameter "windowsCredentialProviderURL"}}" + # CSI Proxy settings $global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}"); $global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}"; @@ -8696,7 +8699,9 @@ try Get-LogCollectionScripts Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - + + Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{ AKSCustomCloudContainerRegistryDNSSuffix }}"{{else}}""{{end}} + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL $cniBinPath = $global:AzureCNIBinDir diff --git a/vhdbuilder/packer/generate-windows-vhd-configuration.ps1 b/vhdbuilder/packer/generate-windows-vhd-configuration.ps1 index 2a032cfa8c8..9a487879a38 100644 --- a/vhdbuilder/packer/generate-windows-vhd-configuration.ps1 +++ b/vhdbuilder/packer/generate-windows-vhd-configuration.ps1 @@ -138,6 +138,9 @@ $global:map = @{ "c:\akse-cache\csi-proxy\" = @( "https://acs-mirror.azureedge.net/csi-proxy/v1.1.2-hotfix.20230807/binaries/csi-proxy-v1.1.2-hotfix.20230807.tar.gz" ); + "c:\akse-cache\credential-provider\" = @( + "https://acs-mirror.azureedge.net/cloud-provider-azure/v1.29.2/binaries/azure-acr-credential-provider-windows-amd64-v1.29.2.tar.gz" + ); # When to remove depracted Kubernetes Windows packages: # There are 30 days grace period before a depracted Kubernetes version is out of supported # xref: https://docs.microsoft.com/en-us/azure/aks/supported-kubernetes-versions diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index f61df83d28a..20e2da02011 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -410,7 +410,7 @@ start_watch # Kubelet credential provider plugins CREDENTIAL_PROVIDER_VERSIONS=" -1.29.3 +1.29.2 " for CREDENTIAL_PROVIDER_VERSION in $CREDENTIAL_PROVIDER_VERSIONS; do CREDENTIAL_PROVIDER_DOWNLOAD_URL="https://acs-mirror.azureedge.net/cloud-provider-azure/v${CREDENTIAL_PROVIDER_VERSION}/binaries/azure-acr-credential-provider-linux-${CPU_ARCH}-v${CREDENTIAL_PROVIDER_VERSION}.tar.gz"