From 577f39c8f27b60e0e1dc178b0f3cebb5494a8ef9 Mon Sep 17 00:00:00 2001 From: Artun Duman Date: Tue, 9 Apr 2024 16:32:02 -0700 Subject: [PATCH 1/3] sig image definition for fips 22.04 --- pkg/agent/datamodel/sig_config.go | 21 +++++++++++++++++++++ pkg/agent/datamodel/sig_config_test.go | 2 +- pkg/agent/datamodel/types.go | 4 ++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index d6ae6796343..3bc6676916d 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -108,6 +108,8 @@ var AvailableUbuntu2204Distros = []Distro{ AKSUbuntuEdgeZoneContainerd2204Gen2, AKSUbuntuMinimalContainerd2204, AKSUbuntuMinimalContainerd2204Gen2, + AKSUbuntuFipsContainerd2204, + AKSUbuntuFipsContainerd2204Gen2, } //nolint:gochecknoglobals @@ -120,6 +122,8 @@ var AvailableContainerdDistros = []Distro{ AKSUbuntuFipsContainerd1804Gen2, AKSUbuntuFipsContainerd2004, AKSUbuntuFipsContainerd2004Gen2, + AKSUbuntuFipsContainerd2204, + AKSUbuntuFipsContainerd2204Gen2, AKSUbuntuEdgeZoneContainerd1804, AKSUbuntuEdgeZoneContainerd1804Gen2, AKSCBLMarinerV1, @@ -165,6 +169,7 @@ var AvailableGen2Distros = []Distro{ AKSUbuntuGPUContainerd1804Gen2, AKSUbuntuFipsContainerd1804Gen2, AKSUbuntuFipsContainerd2004Gen2, + AKSUbuntuFipsContainerd2204Gen2, AKSUbuntuEdgeZoneContainerd1804Gen2, AKSUbuntuArm64Containerd2204Gen2, AKSUbuntuContainerd2204Gen2, @@ -477,6 +482,20 @@ var ( Version: LinuxSIGImageVersion, } + SIGUbuntuFipsContainerd2204ImageConfigTemplate = SigImageConfigTemplate{ + ResourceGroup: AKSUbuntuResourceGroup, + Gallery: AKSUbuntuGalleryName, + Definition: "2204fipscontainerd", + Version: "202404.09.0", // TODO(artunduman): Update version when the image is ready + } + + SIGUbuntuFipsContainerd2204Gen2ImageConfigTemplate = SigImageConfigTemplate{ + ResourceGroup: AKSUbuntuResourceGroup, + Gallery: AKSUbuntuGalleryName, + Definition: "2204gen2fipscontainerd", + Version: "202404.09.0", // TODO(artunduman): Update version when the image is ready + } + SIGUbuntuArm64Containerd2204Gen2ImageConfigTemplate = SigImageConfigTemplate{ ResourceGroup: AKSUbuntuResourceGroup, Gallery: AKSUbuntuGalleryName, @@ -703,6 +722,8 @@ func getSigUbuntuImageConfigMapWithOpts(opts ...SigImageConfigOpt) map[Distro]Si AKSUbuntuFipsContainerd1804Gen2: SIGUbuntuFipsContainerd1804Gen2ImageConfigTemplate.WithOptions(opts...), AKSUbuntuFipsContainerd2004: SIGUbuntuFipsContainerd2004ImageConfigTemplate.WithOptions(opts...), AKSUbuntuFipsContainerd2004Gen2: SIGUbuntuFipsContainerd2004Gen2ImageConfigTemplate.WithOptions(opts...), + AKSUbuntuFipsContainerd2204: SIGUbuntuFipsContainerd2204ImageConfigTemplate.WithOptions(opts...), + AKSUbuntuFipsContainerd2204Gen2: SIGUbuntuFipsContainerd2204Gen2ImageConfigTemplate.WithOptions(opts...), AKSUbuntuContainerd2204: SIGUbuntuContainerd2204ImageConfigTemplate.WithOptions(opts...), AKSUbuntuContainerd2204Gen2: SIGUbuntuContainerd2204Gen2ImageConfigTemplate.WithOptions(opts...), AKSUbuntuContainerd2004CVMGen2: SIGUbuntuContainerd2004CVMGen2ImageConfigTemplate.WithOptions(opts...), diff --git a/pkg/agent/datamodel/sig_config_test.go b/pkg/agent/datamodel/sig_config_test.go index 745a5dd3d5e..f242f14815d 100644 --- a/pkg/agent/datamodel/sig_config_test.go +++ b/pkg/agent/datamodel/sig_config_test.go @@ -47,7 +47,7 @@ var _ = Describe("GetSIGAzureCloudSpecConfig", func() { Expect(sigConfig.SigTenantID).To(Equal("sometenantid")) Expect(sigConfig.SubscriptionID).To(Equal("somesubid")) - Expect(len(sigConfig.SigUbuntuImageConfig)).To(Equal(21)) + Expect(len(sigConfig.SigUbuntuImageConfig)).To(Equal(23)) aksUbuntuGPU1804Gen2 := sigConfig.SigUbuntuImageConfig[AKSUbuntuGPU1804Gen2] Expect(aksUbuntuGPU1804Gen2.ResourceGroup).To(Equal("resourcegroup")) diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index a70e7d52b72..82656bd23ed 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -172,6 +172,8 @@ const ( AKSUbuntuFipsContainerd1804Gen2 Distro = "aks-ubuntu-fips-containerd-18.04-gen2" AKSUbuntuFipsContainerd2004 Distro = "aks-ubuntu-fips-containerd-20.04" AKSUbuntuFipsContainerd2004Gen2 Distro = "aks-ubuntu-fips-containerd-20.04-gen2" + AKSUbuntuFipsContainerd2204 Distro = "aks-ubuntu-fips-containerd-22.04" + AKSUbuntuFipsContainerd2204Gen2 Distro = "aks-ubuntu-fips-containerd-22.04-gen2" AKSUbuntuEdgeZoneContainerd1804 Distro = "aks-ubuntu-edgezone-containerd-18.04" AKSUbuntuEdgeZoneContainerd1804Gen2 Distro = "aks-ubuntu-edgezone-containerd-18.04-gen2" AKSUbuntuEdgeZoneContainerd2204 Distro = "aks-ubuntu-edgezone-containerd-22.04" @@ -246,6 +248,8 @@ var AKSDistrosAvailableOnVHD = []Distro{ AKSUbuntuFipsContainerd1804Gen2, AKSUbuntuFipsContainerd2004, AKSUbuntuFipsContainerd2004Gen2, + AKSUbuntuFipsContainerd2204, + AKSUbuntuFipsContainerd2204Gen2, AKSUbuntuEdgeZoneContainerd1804, AKSUbuntuEdgeZoneContainerd1804Gen2, AKSUbuntuEdgeZoneContainerd2204, From 05d4f6182dfdf1c44f80b08558438aa8093806e7 Mon Sep 17 00:00:00 2001 From: Artun Duman Date: Tue, 9 Apr 2024 16:33:33 -0700 Subject: [PATCH 2/3] go fmt --- pkg/agent/datamodel/sig_config.go | 8 +- pkg/templates/templates_generated.go | 1979 +++++++++++++------------- 2 files changed, 995 insertions(+), 992 deletions(-) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 3bc6676916d..b10cfb18c19 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -486,14 +486,14 @@ var ( ResourceGroup: AKSUbuntuResourceGroup, Gallery: AKSUbuntuGalleryName, Definition: "2204fipscontainerd", - Version: "202404.09.0", // TODO(artunduman): Update version when the image is ready + Version: "202404.09.0", // TODO(artunduman): Update version when the image is ready } SIGUbuntuFipsContainerd2204Gen2ImageConfigTemplate = SigImageConfigTemplate{ ResourceGroup: AKSUbuntuResourceGroup, Gallery: AKSUbuntuGalleryName, Definition: "2204gen2fipscontainerd", - Version: "202404.09.0", // TODO(artunduman): Update version when the image is ready + Version: "202404.09.0", // TODO(artunduman): Update version when the image is ready } SIGUbuntuArm64Containerd2204Gen2ImageConfigTemplate = SigImageConfigTemplate{ @@ -722,8 +722,8 @@ func getSigUbuntuImageConfigMapWithOpts(opts ...SigImageConfigOpt) map[Distro]Si AKSUbuntuFipsContainerd1804Gen2: SIGUbuntuFipsContainerd1804Gen2ImageConfigTemplate.WithOptions(opts...), AKSUbuntuFipsContainerd2004: SIGUbuntuFipsContainerd2004ImageConfigTemplate.WithOptions(opts...), AKSUbuntuFipsContainerd2004Gen2: SIGUbuntuFipsContainerd2004Gen2ImageConfigTemplate.WithOptions(opts...), - AKSUbuntuFipsContainerd2204: SIGUbuntuFipsContainerd2204ImageConfigTemplate.WithOptions(opts...), - AKSUbuntuFipsContainerd2204Gen2: SIGUbuntuFipsContainerd2204Gen2ImageConfigTemplate.WithOptions(opts...), + AKSUbuntuFipsContainerd2204: SIGUbuntuFipsContainerd2204ImageConfigTemplate.WithOptions(opts...), + AKSUbuntuFipsContainerd2204Gen2: SIGUbuntuFipsContainerd2204Gen2ImageConfigTemplate.WithOptions(opts...), AKSUbuntuContainerd2204: SIGUbuntuContainerd2204ImageConfigTemplate.WithOptions(opts...), AKSUbuntuContainerd2204Gen2: SIGUbuntuContainerd2204Gen2ImageConfigTemplate.WithOptions(opts...), AKSUbuntuContainerd2004CVMGen2: SIGUbuntuContainerd2004CVMGen2ImageConfigTemplate.WithOptions(opts...), diff --git a/pkg/templates/templates_generated.go b/pkg/templates/templates_generated.go index 4215ab52247..bbd294b4563 100644 --- a/pkg/templates/templates_generated.go +++ b/pkg/templates/templates_generated.go @@ -114,6 +114,7 @@ import ( "strings" "time" ) + type asset struct { bytes []byte info os.FileInfo @@ -3150,7 +3151,7 @@ ERR_AZURE_STACK_GET_ARM_TOKEN=120 # Error generating a token to use with Azure R ERR_AZURE_STACK_GET_NETWORK_CONFIGURATION=121 # Error fetching the network configuration for the node ERR_AZURE_STACK_GET_SUBNET_PREFIX=122 # Error fetching the subnet address prefix for a subnet ID -# Error code 124 is returned when a `+"`"+`timeout`+"`"+` command times out, and --preserve-status is not specified: https://man7.org/linux/man-pages/man1/timeout.1.html +# Error code 124 is returned when a ` + "`" + `timeout` + "`" + ` command times out, and --preserve-status is not specified: https://man7.org/linux/man-pages/man1/timeout.1.html ERR_VHD_BUILD_ERROR=125 # Reserved for VHD CI exit conditions ERR_SWAP_CREATE_FAIL=130 # Error allocating swap file @@ -3186,7 +3187,7 @@ MARINER_OS_NAME="MARINER" KUBECTL=/usr/local/bin/kubectl DOCKER=/usr/bin/docker # this will be empty during VHD build -# but vhd build runs with `+"`"+`set -o nounset`+"`"+` +# but vhd build runs with ` + "`" + `set -o nounset` + "`" + ` # so needs a default value # prefer empty string to avoid potential "it works but did something weird" scenarios export GPU_DV="${GPU_DRIVER_VERSION:=}" @@ -3795,7 +3796,7 @@ installKubeletKubectlAndKubeProxy() { # remove the kubelet and kubectl binaries to make sure the only binary left is from the CUSTOM_KUBE_BINARY_DOWNLOAD_URL rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* - # NOTE(mainred): we expect kubelet binary to be under `+"`"+`kubernetes/node/bin`+"`"+`. This suits the current setting of + # NOTE(mainred): we expect kubelet binary to be under ` + "`" + `kubernetes/node/bin` + "`" + `. This suits the current setting of # kube binaries used by AKS and Kubernetes upstream. # TODO(mainred): let's see if necessary to auto-detect the path of kubelet logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL} false @@ -4168,8 +4169,8 @@ EOF if [[ "${MIG_NODE}" == "true" ]]; then # A100 GPU has a bit in the physical card (infoROM) to enable mig mode. # Changing this bit in either direction requires a VM reboot on Azure (hypervisor/plaform stuff). - # Commands such as `+"`"+`nvidia-smi --gpu-reset`+"`"+` may succeed, - # while commands such as `+"`"+`nvidia-smi -q`+"`"+` will show mismatched current/pending mig mode. + # Commands such as ` + "`" + `nvidia-smi --gpu-reset` + "`" + ` may succeed, + # while commands such as ` + "`" + `nvidia-smi -q` + "`" + ` will show mismatched current/pending mig mode. # this will not be required per nvidia for next gen H100. REBOOTREQUIRED=true @@ -6184,7 +6185,7 @@ var _linuxCloudInitArtifactsPamDCommonPassword = []byte(`# # The "sha512" option enables salted SHA512 passwords. Without this option, # the default is Unix crypt. Prior releases used the option "md5". # -# The "obscure" option replaces the old `+"`"+`OBSCURE_CHECKS_ENAB' option in +# The "obscure" option replaces the old ` + "`" + `OBSCURE_CHECKS_ENAB' option in # login.defs. # # See the pam_unix manpage for other options. @@ -6228,19 +6229,19 @@ func linuxCloudInitArtifactsPamDCommonPassword() (*asset, error) { } var _linuxCloudInitArtifactsPamDSu = []byte(`# -# The PAM configuration file for the Shadow `+"`"+`su' service +# The PAM configuration file for the Shadow ` + "`" + `su' service # # This allows root to su without passwords (normal operation) auth sufficient pam_rootok.so # Uncomment this to force users to be a member of group root -# before they can use `+"`"+`su'. You can also add "group=foo" +# before they can use ` + "`" + `su'. You can also add "group=foo" # to the end of this line if you want to use a group other # than the default "root" (but this may have side effect of # denying "root" user, unless she's a member of "foo" or explicitly # permitted earlier by e.g. "sufficient pam_rootok.so"). -# (Replaces the `+"`"+`SU_WHEEL_ONLY' option from login.defs) +# (Replaces the ` + "`" + `SU_WHEEL_ONLY' option from login.defs) # 5.6 Ensure access to the su command is restricted auth required pam_wheel.so use_uid @@ -6255,7 +6256,7 @@ auth required pam_wheel.so use_uid # Uncomment and edit /etc/security/time.conf if you need to set # time restrainst on su usage. -# (Replaces the `+"`"+`PORTTIME_CHECKS_ENAB' option from login.defs +# (Replaces the ` + "`" + `PORTTIME_CHECKS_ENAB' option from login.defs # as well as /etc/porttime) # account requisite pam_time.so @@ -8323,519 +8324,519 @@ func windowsCsecmdPs1() (*asset, error) { return a, nil } -var _windowsKuberneteswindowssetupPs1 = []byte(`<# - .SYNOPSIS - Provisions VM as a Kubernetes agent. - - .DESCRIPTION - Provisions VM as a Kubernetes agent. - - The parameters passed in are required, and will vary per-deployment. - - Notes on modifying this file: - - This file extension is PS1, but it is actually used as a template from pkg/engine/template_generator.go - - All of the lines that have braces in them will be modified. Please do not change them here, change them in the Go sources - - Single quotes are forbidden, they are reserved to delineate the different members for the ARM template concat() call - - windowscsehelper.ps1 contains basic util functions. It will be compressed to a zip file and then be converted to base64 encoding - string and stored in $zippedFiles. Reason: This script is a template and has some limitations. - - All other scripts will be packaged and published in a single package. It will be downloaded in provisioning VM. - Reason: CustomData has length limitation 87380. - - ProvisioningScriptsPackage contains scripts to start kubelet, kubeproxy, etc. The source is https://github.com/Azure/aks-engine/tree/master/staging/provisioning/windows -#> -[CmdletBinding(DefaultParameterSetName="Standard")] -param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AgentKey, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientSecret, # base64 - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $LogFile, - - # C:\AzureData\provision.complete - # MUST keep generating this file when CSE is done and do not change the name - # - It is used to avoid running CSE multiple times - # - Some customers use this file to check if CSE is done - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID -) -# Do not parse the start time from $LogFile to simplify the logic -$StartTime=Get-Date -$global:ExitCode=0 -$global:ErrorMessage="" -Start-Transcript -Path $LogFile -# These globals will not change between nodes in the same cluster, so they are not -# passed as powershell parameters - -## SSH public keys to add to authorized_keys -$global:SSHKeys = @( {{ GetSshPublicKeysPowerShell }} ) - -## Certificates generated by aks-engine -$global:CACertificate = "{{GetParameter "caCertificate"}}" -$global:AgentCertificate = "{{GetParameter "clientCertificate"}}" - -## Download sources provided by aks-engine -$global:KubeBinariesPackageSASURL = "{{GetParameter "kubeBinariesSASURL"}}" -$global:WindowsKubeBinariesURL = "{{GetParameter "windowsKubeBinariesURL"}}" -$global:KubeBinariesVersion = "{{GetParameter "kubeBinariesVersion"}}" -$global:ContainerdUrl = "{{GetParameter "windowsContainerdURL"}}" -$global:ContainerdSdnPluginUrl = "{{GetParameter "windowsSdnPluginURL"}}" - -## Docker Version -$global:DockerVersion = "{{GetParameter "windowsDockerVersion"}}" - -## ContainerD Usage -$global:DefaultContainerdWindowsSandboxIsolation = "{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}" -$global:ContainerdWindowsRuntimeHandlers = "{{GetParameter "containerdWindowsRuntimeHandlers"}}" - -## VM configuration passed by Azure -$global:WindowsTelemetryGUID = "{{GetParameter "windowsTelemetryGUID"}}" -{{if eq GetIdentitySystem "adfs"}} -$global:TenantId = "adfs" -{{else}} -$global:TenantId = "{{GetVariable "tenantID"}}" -{{end}} -$global:SubscriptionId = "{{GetVariable "subscriptionId"}}" -$global:ResourceGroup = "{{GetVariable "resourceGroup"}}" -$global:VmType = "{{GetVariable "vmType"}}" -$global:SubnetName = "{{GetVariable "subnetName"}}" -# NOTE: MasterSubnet is still referenced by `+"`"+`kubeletstart.ps1`+"`"+` and `+"`"+`windowsnodereset.ps1`+"`"+` -# for case of Kubenet -$global:MasterSubnet = "" -$global:SecurityGroupName = "{{GetVariable "nsgName"}}" -$global:VNetName = "{{GetVariable "virtualNetworkName"}}" -$global:RouteTableName = "{{GetVariable "routeTableName"}}" -$global:PrimaryAvailabilitySetName = "{{GetVariable "primaryAvailabilitySetName"}}" -$global:PrimaryScaleSetName = "{{GetVariable "primaryScaleSetName"}}" - -$global:KubeClusterCIDR = "{{GetParameter "kubeClusterCidr"}}" -$global:KubeServiceCIDR = "{{GetParameter "kubeServiceCidr"}}" -$global:VNetCIDR = "{{GetParameter "vnetCidr"}}" -{{if IsKubernetesVersionGe "1.16.0"}} -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . }}" -{{else}} -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabelsDeprecated . }}" -{{end}} -$global:KubeletConfigArgs = @( {{GetKubeletConfigKeyValsPsh}} ) -$global:KubeproxyConfigArgs = @( {{GetKubeproxyConfigKeyValsPsh}} ) - -$global:KubeproxyFeatureGates = @( {{GetKubeProxyFeatureGatesPsh}} ) - -$global:UseManagedIdentityExtension = "{{GetVariable "useManagedIdentityExtension"}}" -$global:UseInstanceMetadata = "{{GetVariable "useInstanceMetadata"}}" - -$global:LoadBalancerSku = "{{GetVariable "loadBalancerSku"}}" -$global:ExcludeMasterFromStandardLB = "{{GetVariable "excludeMasterFromStandardLB"}}" - -$global:PrivateEgressProxyAddress = "{{GetPrivateEgressProxyAddress}}" - -# Windows defaults, not changed by aks-engine -$global:CacheDir = "c:\akse-cache" -$global:KubeDir = "c:\k" -$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") - -$global:KubeDnsSearchPath = "svc.cluster.local" - -$global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") -$global:NetworkMode = "L2Bridge" -$global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`+"`"+`$global:NetworkMode.conf") -$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") - - -$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") -$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") -$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") - -# Azure cni configuration -# $global:NetworkPolicy = "{{GetParameter "networkPolicy"}}" # BUG: unused -$global:NetworkPlugin = "{{GetParameter "networkPlugin"}}" -$global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" -$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} -$global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} - -# CSI Proxy settings -$global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}"); -$global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}"; - -# Hosts Config Agent settings -$global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("{{ EnableHostsConfigAgent }}"); - -# These scripts are used by cse -$global:CSEScriptsPackageUrl = "{{GetVariable "windowsCSEScriptsPackageURL" }}"; - -# The windows nvidia gpu driver related url is used by windows cse -$global:GpuDriverURL = "{{GetVariable "windowsGpuDriverURL" }}"; - -# PauseImage -$global:WindowsPauseImageURL = "{{GetVariable "windowsPauseImageURL" }}"; -$global:AlwaysPullWindowsPauseImage = [System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage" }}"); - -# Calico -$global:WindowsCalicoPackageURL = "{{GetVariable "windowsCalicoPackageURL" }}"; - -## GPU install -$global:ConfigGPUDriverIfNeeded = [System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded" }}"); - -# GMSA -$global:WindowsGmsaPackageUrl = "{{GetVariable "windowsGmsaPackageUrl" }}"; - -# TLS Bootstrap Token -$global:TLSBootstrapToken = "{{GetTLSBootstrapTokenForKubeConfig}}" - -# Disable OutBoundNAT in Azure CNI configuration -$global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat" }}"); - -# Base64 representation of ZIP archive -$zippedFiles = "{{ GetKubernetesWindowsAgentFunctions }}" - -$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" -$fipsEnabled = [System.Convert]::ToBoolean("{{ FIPSEnabled }}") - -# HNS remediator -$global:HNSRemediatorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}"); - -# Log generator -$global:LogGeneratorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}"); - -$global:EnableIncreaseDynamicPortRange = $false - -$global:RebootNeeded = $false - -# Extract cse helper script from ZIP -[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) -Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force - -# Dot-source windowscsehelper.ps1 with functions that are called in this script -. c:\AzureData\windows\windowscsehelper.ps1 -# util functions only can be used after this line, for example, Write-Log - -$global:OperationId = New-Guid - -try -{ - Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" - - # Exit early if the script has been executed - if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { - Write-Log "The script has been executed before, will exit without doing anything." - return - } - - # This involes using proxy, log the config before fetching packages - Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'" - # TODO update to use proxy - - $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.40.zip" - Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" - Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" - # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl - # but it is better to set the CSE package version in Windows CSE in AgentBaker - # since most changes in CSE package also need the change in Windows CSE in AgentBaker - # In future, AKS RP only sets the endpoint with the pacakge name, for example, https://acs-mirror.azureedge.net/aks/windows/cse/ - if ($global:CSEScriptsPackageUrl.EndsWith("/")) { - $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage - Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" - } - - # Download CSE function scripts - Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" - $tempfile = 'c:\csescripts.zip' - DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE - Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" -Force - Remove-Item -Path $tempfile -Force - - # Dot-source cse scripts with functions that are called in this script - . c:\AzureData\windows\azurecnifunc.ps1 - . c:\AzureData\windows\calicofunc.ps1 - . c:\AzureData\windows\configfunc.ps1 - . c:\AzureData\windows\containerdfunc.ps1 - . c:\AzureData\windows\kubeletfunc.ps1 - . c:\AzureData\windows\kubernetesfunc.ps1 - . c:\AzureData\windows\nvidiagpudriverfunc.ps1 - - # Install OpenSSH if SSH enabled - $sshEnabled = [System.Convert]::ToBoolean("{{ WindowsSSHEnabled }}") - - if ( $sshEnabled ) { - Install-OpenSSH -SSHKeys $SSHKeys - } - - Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - - Resize-OSDrive - - Initialize-DataDisks - - Initialize-DataDirectories - - Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" - Create-Directory -FullPath "c:\k" - Write-Log "Remove `+"`"+`"NT AUTHORITY\Authenticated Users`+"`"+`" write permissions on files in c:\k" - icacls.exe "c:\k" /inheritance:r - icacls.exe "c:\k" /grant:r SYSTEM:`+"`"+`(OI`+"`"+`)`+"`"+`(CI`+"`"+`)`+"`"+`(F`+"`"+`) - icacls.exe "c:\k" /grant:r BUILTIN\Administrators:`+"`"+`(OI`+"`"+`)`+"`"+`(CI`+"`"+`)`+"`"+`(F`+"`"+`) - icacls.exe "c:\k" /grant:r BUILTIN\Users:`+"`"+`(OI`+"`"+`)`+"`"+`(CI`+"`"+`)`+"`"+`(RX`+"`"+`) - Write-Log "c:\k permissions: " - icacls.exe "c:\k" - Get-ProvisioningScripts - Get-LogCollectionScripts - - Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - $cniBinPath = $global:AzureCNIBinDir - $cniConfigPath = $global:AzureCNIConfDir - if ($global:NetworkPlugin -eq "kubenet") { - $cniBinPath = $global:CNIPath - $cniConfigPath = $global:CNIConfigPath - } - - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion - - Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - - # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-AzureConfig `+"`"+` - -KubeDir $global:KubeDir `+"`"+` - -AADClientId $AADClientId `+"`"+` - -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) `+"`"+` - -TenantId $global:TenantId `+"`"+` - -SubscriptionId $global:SubscriptionId `+"`"+` - -ResourceGroup $global:ResourceGroup `+"`"+` - -Location $Location `+"`"+` - -VmType $global:VmType `+"`"+` - -SubnetName $global:SubnetName `+"`"+` - -SecurityGroupName $global:SecurityGroupName `+"`"+` - -VNetName $global:VNetName `+"`"+` - -RouteTableName $global:RouteTableName `+"`"+` - -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName `+"`"+` - -PrimaryScaleSetName $global:PrimaryScaleSetName `+"`"+` - -UseManagedIdentityExtension $global:UseManagedIdentityExtension `+"`"+` - -UserAssignedClientID $UserAssignedClientID `+"`"+` - -UseInstanceMetadata $global:UseInstanceMetadata `+"`"+` - -LoadBalancerSku $global:LoadBalancerSku `+"`"+` - -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB `+"`"+` - -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud"{{else}}$TargetEnvironment{{end}} - - # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud. - # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - {{if IsAKSCustomCloud}} - $azureStackConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") - $envJSON = "{{ GetBase64EncodedEnvironmentJSON }}" - [io.file]::WriteAllBytes($azureStackConfigFile, [System.Convert]::FromBase64String($envJSON)) - - Get-CACertificates - {{end}} - - Write-CACert -CACertificate $global:CACertificate `+"`"+` - -KubeDir $global:KubeDir - - if ($global:EnableCsiProxy) { - New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir - } - - if ($global:TLSBootstrapToken) { - Write-BootstrapKubeConfig -CACertificate $global:CACertificate `+"`"+` - -KubeDir $global:KubeDir `+"`"+` - -MasterFQDNPrefix $MasterFQDNPrefix `+"`"+` - -MasterIP $MasterIP `+"`"+` - -TLSBootstrapToken $global:TLSBootstrapToken - - # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled - # This kubeconfig will deleted after calico installation. - # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block - Write-Log "Write temporary kube config" - } else { - Write-Log "Write kube config" - } - - Write-KubeConfig -CACertificate $global:CACertificate `+"`"+` - -KubeDir $global:KubeDir `+"`"+` - -MasterFQDNPrefix $MasterFQDNPrefix `+"`"+` - -MasterIP $MasterIP `+"`"+` - -AgentKey $AgentKey `+"`"+` - -AgentCertificate $global:AgentCertificate - - if ($global:EnableHostsConfigAgent) { - New-HostsConfigService - } - - Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin" - - # Configure network policy. - Get-HnsPsm1 -HNSModule $global:HNSModule - Import-Module $global:HNSModule - - Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir `+"`"+` - -AzureCNIBinDir $global:AzureCNIBinDir `+"`"+` - -VNetCNIPluginsURL $global:VNetCNIPluginsURL - - Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir `+"`"+` - -KubeDnsSearchPath $global:KubeDnsSearchPath `+"`"+` - -KubeClusterCIDR $global:KubeClusterCIDR `+"`"+` - -KubeServiceCIDR $global:KubeServiceCIDR `+"`"+` - -VNetCIDR $global:VNetCIDR `+"`"+` - -IsDualStackEnabled $global:IsDualStackEnabled `+"`"+` - -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - - if ($TargetEnvironment -ieq "AzureStackCloud") { - GenerateAzureStackCNIConfig `+"`"+` - -TenantId $global:TenantId `+"`"+` - -SubscriptionId $global:SubscriptionId `+"`"+` - -ResourceGroup $global:ResourceGroup `+"`"+` - -AADClientId $AADClientId `+"`"+` - -KubeDir $global:KubeDir `+"`"+` - -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) `+"`"+` - -NetworkAPIVersion $NetworkAPIVersion `+"`"+` - -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) `+"`"+` - -IdentitySystem "{{ GetIdentitySystem }}" - } - - New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - - Install-KubernetesServices `+"`"+` - -KubeDir $global:KubeDir - - Set-Explorer - Adjust-PageFileSize - Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - PREPROVISION_EXTENSION - Update-ServiceFailureActions - Adjust-DynamicPortRange - Register-LogsCleanupScriptTask - Register-NodeResetScriptTask - Update-DefenderPreferences - - $windowsVersion = Get-WindowsVersion - if ($windowsVersion -ne "1809") { - Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" - } else { - Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" - try { - . C:\k\windowssecuretls.ps1 - Enable-SecureTls - } - catch { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS -ErrorMessage $_ - } - } - - Enable-FIPSMode -FipsEnabled $fipsEnabled - if ($global:WindowsGmsaPackageUrl) { - Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl - } - - Check-APIServerConnectivity -MasterIP $MasterIP - - if ($global:WindowsCalicoPackageURL) { - Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp - } - - Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - - if (Test-Path $CacheDir) - { - Write-Log "Removing aks cache directory" - Remove-Item $CacheDir -Recurse -Force - } - - if ($global:TLSBootstrapToken) { - Write-Log "Removing temporary kube config" - $kubeConfigFile = [io.path]::Combine($KubeDir, "config") - Remove-Item $kubeConfigFile - } - - Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - if ($global:RebootNeeded) { - Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot" - Postpone-RestartComputer - } else { - Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" - Start-ScheduledTask -TaskName "k8s-restart-job" - - $timeout = 180 ## seconds - $timer = [Diagnostics.Stopwatch]::StartNew() - while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') { - # The task `+"`"+`k8s-restart-job`+"`"+` needs ~8 seconds. - if ($timer.Elapsed.TotalSeconds -gt $timeout) { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds" - } - - Write-Log -Message "Waiting on NodeResetScriptTask..." - Start-Sleep -Seconds 3 - } - $timer.Stop() - Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" - } -} -catch -{ - # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block - # Ideally all exceptions will be handled and no exception will be thrown. - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_ -} -finally -{ - # Generate CSE result so it can be returned as the CSE response in csecmd.ps1 - $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date)) - Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode" - # $CSEResultFilePath is used to avoid running CSE multiple times - Set-Content -Path $CSEResultFilePath -Value $global:ExitCode -Force - Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." - # Please not use Write-Log or Logs-To-Events after Stop-Transcript - Stop-Transcript - - # Remove the parameters in the log file to avoid leaking secrets - $logs=Get-Content $LogFile | Where-Object {$_ -notmatch "^Host Application: "} - $logs | Set-Content $LogFile - - Upload-GuestVMLogs -ExitCode $global:ExitCode - if ($global:ExitCode -ne 0) { - # $JsonString = "ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" - # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe. - $errorMessageLength = "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length - $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) - throw "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|" - } -} +var _windowsKuberneteswindowssetupPs1 = []byte(`<# + .SYNOPSIS + Provisions VM as a Kubernetes agent. + + .DESCRIPTION + Provisions VM as a Kubernetes agent. + + The parameters passed in are required, and will vary per-deployment. + + Notes on modifying this file: + - This file extension is PS1, but it is actually used as a template from pkg/engine/template_generator.go + - All of the lines that have braces in them will be modified. Please do not change them here, change them in the Go sources + - Single quotes are forbidden, they are reserved to delineate the different members for the ARM template concat() call + - windowscsehelper.ps1 contains basic util functions. It will be compressed to a zip file and then be converted to base64 encoding + string and stored in $zippedFiles. Reason: This script is a template and has some limitations. + - All other scripts will be packaged and published in a single package. It will be downloaded in provisioning VM. + Reason: CustomData has length limitation 87380. + - ProvisioningScriptsPackage contains scripts to start kubelet, kubeproxy, etc. The source is https://github.com/Azure/aks-engine/tree/master/staging/provisioning/windows +#> +[CmdletBinding(DefaultParameterSetName="Standard")] +param( + [string] + [ValidateNotNullOrEmpty()] + $MasterIP, + + [parameter()] + [ValidateNotNullOrEmpty()] + $KubeDnsServiceIp, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $MasterFQDNPrefix, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $Location, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AgentKey, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AADClientId, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AADClientSecret, # base64 + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $NetworkAPIVersion, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $TargetEnvironment, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $LogFile, + + # C:\AzureData\provision.complete + # MUST keep generating this file when CSE is done and do not change the name + # - It is used to avoid running CSE multiple times + # - Some customers use this file to check if CSE is done + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $CSEResultFilePath, + + [string] + $UserAssignedClientID +) +# Do not parse the start time from $LogFile to simplify the logic +$StartTime=Get-Date +$global:ExitCode=0 +$global:ErrorMessage="" +Start-Transcript -Path $LogFile +# These globals will not change between nodes in the same cluster, so they are not +# passed as powershell parameters + +## SSH public keys to add to authorized_keys +$global:SSHKeys = @( {{ GetSshPublicKeysPowerShell }} ) + +## Certificates generated by aks-engine +$global:CACertificate = "{{GetParameter "caCertificate"}}" +$global:AgentCertificate = "{{GetParameter "clientCertificate"}}" + +## Download sources provided by aks-engine +$global:KubeBinariesPackageSASURL = "{{GetParameter "kubeBinariesSASURL"}}" +$global:WindowsKubeBinariesURL = "{{GetParameter "windowsKubeBinariesURL"}}" +$global:KubeBinariesVersion = "{{GetParameter "kubeBinariesVersion"}}" +$global:ContainerdUrl = "{{GetParameter "windowsContainerdURL"}}" +$global:ContainerdSdnPluginUrl = "{{GetParameter "windowsSdnPluginURL"}}" + +## Docker Version +$global:DockerVersion = "{{GetParameter "windowsDockerVersion"}}" + +## ContainerD Usage +$global:DefaultContainerdWindowsSandboxIsolation = "{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}" +$global:ContainerdWindowsRuntimeHandlers = "{{GetParameter "containerdWindowsRuntimeHandlers"}}" + +## VM configuration passed by Azure +$global:WindowsTelemetryGUID = "{{GetParameter "windowsTelemetryGUID"}}" +{{if eq GetIdentitySystem "adfs"}} +$global:TenantId = "adfs" +{{else}} +$global:TenantId = "{{GetVariable "tenantID"}}" +{{end}} +$global:SubscriptionId = "{{GetVariable "subscriptionId"}}" +$global:ResourceGroup = "{{GetVariable "resourceGroup"}}" +$global:VmType = "{{GetVariable "vmType"}}" +$global:SubnetName = "{{GetVariable "subnetName"}}" +# NOTE: MasterSubnet is still referenced by ` + "`" + `kubeletstart.ps1` + "`" + ` and ` + "`" + `windowsnodereset.ps1` + "`" + ` +# for case of Kubenet +$global:MasterSubnet = "" +$global:SecurityGroupName = "{{GetVariable "nsgName"}}" +$global:VNetName = "{{GetVariable "virtualNetworkName"}}" +$global:RouteTableName = "{{GetVariable "routeTableName"}}" +$global:PrimaryAvailabilitySetName = "{{GetVariable "primaryAvailabilitySetName"}}" +$global:PrimaryScaleSetName = "{{GetVariable "primaryScaleSetName"}}" + +$global:KubeClusterCIDR = "{{GetParameter "kubeClusterCidr"}}" +$global:KubeServiceCIDR = "{{GetParameter "kubeServiceCidr"}}" +$global:VNetCIDR = "{{GetParameter "vnetCidr"}}" +{{if IsKubernetesVersionGe "1.16.0"}} +$global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . }}" +{{else}} +$global:KubeletNodeLabels = "{{GetAgentKubernetesLabelsDeprecated . }}" +{{end}} +$global:KubeletConfigArgs = @( {{GetKubeletConfigKeyValsPsh}} ) +$global:KubeproxyConfigArgs = @( {{GetKubeproxyConfigKeyValsPsh}} ) + +$global:KubeproxyFeatureGates = @( {{GetKubeProxyFeatureGatesPsh}} ) + +$global:UseManagedIdentityExtension = "{{GetVariable "useManagedIdentityExtension"}}" +$global:UseInstanceMetadata = "{{GetVariable "useInstanceMetadata"}}" + +$global:LoadBalancerSku = "{{GetVariable "loadBalancerSku"}}" +$global:ExcludeMasterFromStandardLB = "{{GetVariable "excludeMasterFromStandardLB"}}" + +$global:PrivateEgressProxyAddress = "{{GetPrivateEgressProxyAddress}}" + +# Windows defaults, not changed by aks-engine +$global:CacheDir = "c:\akse-cache" +$global:KubeDir = "c:\k" +$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") + +$global:KubeDnsSearchPath = "svc.cluster.local" + +$global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") +$global:NetworkMode = "L2Bridge" +$global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "` + "`" + `$global:NetworkMode.conf") +$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") + + +$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") +$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") +$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") + +# Azure cni configuration +# $global:NetworkPolicy = "{{GetParameter "networkPolicy"}}" # BUG: unused +$global:NetworkPlugin = "{{GetParameter "networkPlugin"}}" +$global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" +$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} +$global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} + +# CSI Proxy settings +$global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}"); +$global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}"; + +# Hosts Config Agent settings +$global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("{{ EnableHostsConfigAgent }}"); + +# These scripts are used by cse +$global:CSEScriptsPackageUrl = "{{GetVariable "windowsCSEScriptsPackageURL" }}"; + +# The windows nvidia gpu driver related url is used by windows cse +$global:GpuDriverURL = "{{GetVariable "windowsGpuDriverURL" }}"; + +# PauseImage +$global:WindowsPauseImageURL = "{{GetVariable "windowsPauseImageURL" }}"; +$global:AlwaysPullWindowsPauseImage = [System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage" }}"); + +# Calico +$global:WindowsCalicoPackageURL = "{{GetVariable "windowsCalicoPackageURL" }}"; + +## GPU install +$global:ConfigGPUDriverIfNeeded = [System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded" }}"); + +# GMSA +$global:WindowsGmsaPackageUrl = "{{GetVariable "windowsGmsaPackageUrl" }}"; + +# TLS Bootstrap Token +$global:TLSBootstrapToken = "{{GetTLSBootstrapTokenForKubeConfig}}" + +# Disable OutBoundNAT in Azure CNI configuration +$global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat" }}"); + +# Base64 representation of ZIP archive +$zippedFiles = "{{ GetKubernetesWindowsAgentFunctions }}" + +$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" +$fipsEnabled = [System.Convert]::ToBoolean("{{ FIPSEnabled }}") + +# HNS remediator +$global:HNSRemediatorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}"); + +# Log generator +$global:LogGeneratorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}"); + +$global:EnableIncreaseDynamicPortRange = $false + +$global:RebootNeeded = $false + +# Extract cse helper script from ZIP +[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) +Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force + +# Dot-source windowscsehelper.ps1 with functions that are called in this script +. c:\AzureData\windows\windowscsehelper.ps1 +# util functions only can be used after this line, for example, Write-Log + +$global:OperationId = New-Guid + +try +{ + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + + # Exit early if the script has been executed + if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { + Write-Log "The script has been executed before, will exit without doing anything." + return + } + + # This involes using proxy, log the config before fetching packages + Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'" + # TODO update to use proxy + + $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.40.zip" + Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" + Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" + # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl + # but it is better to set the CSE package version in Windows CSE in AgentBaker + # since most changes in CSE package also need the change in Windows CSE in AgentBaker + # In future, AKS RP only sets the endpoint with the pacakge name, for example, https://acs-mirror.azureedge.net/aks/windows/cse/ + if ($global:CSEScriptsPackageUrl.EndsWith("/")) { + $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage + Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" + } + + # Download CSE function scripts + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" + $tempfile = 'c:\csescripts.zip' + DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE + Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" -Force + Remove-Item -Path $tempfile -Force + + # Dot-source cse scripts with functions that are called in this script + . c:\AzureData\windows\azurecnifunc.ps1 + . c:\AzureData\windows\calicofunc.ps1 + . c:\AzureData\windows\configfunc.ps1 + . c:\AzureData\windows\containerdfunc.ps1 + . c:\AzureData\windows\kubeletfunc.ps1 + . c:\AzureData\windows\kubernetesfunc.ps1 + . c:\AzureData\windows\nvidiagpudriverfunc.ps1 + + # Install OpenSSH if SSH enabled + $sshEnabled = [System.Convert]::ToBoolean("{{ WindowsSSHEnabled }}") + + if ( $sshEnabled ) { + Install-OpenSSH -SSHKeys $SSHKeys + } + + Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID + + Resize-OSDrive + + Initialize-DataDisks + + Initialize-DataDirectories + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" + Create-Directory -FullPath "c:\k" + Write-Log "Remove ` + "`" + `"NT AUTHORITY\Authenticated Users` + "`" + `" write permissions on files in c:\k" + icacls.exe "c:\k" /inheritance:r + icacls.exe "c:\k" /grant:r SYSTEM:` + "`" + `(OI` + "`" + `)` + "`" + `(CI` + "`" + `)` + "`" + `(F` + "`" + `) + icacls.exe "c:\k" /grant:r BUILTIN\Administrators:` + "`" + `(OI` + "`" + `)` + "`" + `(CI` + "`" + `)` + "`" + `(F` + "`" + `) + icacls.exe "c:\k" /grant:r BUILTIN\Users:` + "`" + `(OI` + "`" + `)` + "`" + `(CI` + "`" + `)` + "`" + `(RX` + "`" + `) + Write-Log "c:\k permissions: " + icacls.exe "c:\k" + Get-ProvisioningScripts + Get-LogCollectionScripts + + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp + + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL + + $cniBinPath = $global:AzureCNIBinDir + $cniConfigPath = $global:AzureCNIConfDir + if ($global:NetworkPlugin -eq "kubenet") { + $cniBinPath = $global:CNIPath + $cniConfigPath = $global:CNIConfigPath + } + + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment + + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud + Write-AzureConfig ` + "`" + ` + -KubeDir $global:KubeDir ` + "`" + ` + -AADClientId $AADClientId ` + "`" + ` + -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` + "`" + ` + -TenantId $global:TenantId ` + "`" + ` + -SubscriptionId $global:SubscriptionId ` + "`" + ` + -ResourceGroup $global:ResourceGroup ` + "`" + ` + -Location $Location ` + "`" + ` + -VmType $global:VmType ` + "`" + ` + -SubnetName $global:SubnetName ` + "`" + ` + -SecurityGroupName $global:SecurityGroupName ` + "`" + ` + -VNetName $global:VNetName ` + "`" + ` + -RouteTableName $global:RouteTableName ` + "`" + ` + -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName ` + "`" + ` + -PrimaryScaleSetName $global:PrimaryScaleSetName ` + "`" + ` + -UseManagedIdentityExtension $global:UseManagedIdentityExtension ` + "`" + ` + -UserAssignedClientID $UserAssignedClientID ` + "`" + ` + -UseInstanceMetadata $global:UseInstanceMetadata ` + "`" + ` + -LoadBalancerSku $global:LoadBalancerSku ` + "`" + ` + -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB ` + "`" + ` + -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud"{{else}}$TargetEnvironment{{end}} + + # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud. + # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json + {{if IsAKSCustomCloud}} + $azureStackConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") + $envJSON = "{{ GetBase64EncodedEnvironmentJSON }}" + [io.file]::WriteAllBytes($azureStackConfigFile, [System.Convert]::FromBase64String($envJSON)) + + Get-CACertificates + {{end}} + + Write-CACert -CACertificate $global:CACertificate ` + "`" + ` + -KubeDir $global:KubeDir + + if ($global:EnableCsiProxy) { + New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir + } + + if ($global:TLSBootstrapToken) { + Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` + "`" + ` + -KubeDir $global:KubeDir ` + "`" + ` + -MasterFQDNPrefix $MasterFQDNPrefix ` + "`" + ` + -MasterIP $MasterIP ` + "`" + ` + -TLSBootstrapToken $global:TLSBootstrapToken + + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled + # This kubeconfig will deleted after calico installation. + # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block + Write-Log "Write temporary kube config" + } else { + Write-Log "Write kube config" + } + + Write-KubeConfig -CACertificate $global:CACertificate ` + "`" + ` + -KubeDir $global:KubeDir ` + "`" + ` + -MasterFQDNPrefix $MasterFQDNPrefix ` + "`" + ` + -MasterIP $MasterIP ` + "`" + ` + -AgentKey $AgentKey ` + "`" + ` + -AgentCertificate $global:AgentCertificate + + if ($global:EnableHostsConfigAgent) { + New-HostsConfigService + } + + Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin" + + # Configure network policy. + Get-HnsPsm1 -HNSModule $global:HNSModule + Import-Module $global:HNSModule + + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` + "`" + ` + -AzureCNIBinDir $global:AzureCNIBinDir ` + "`" + ` + -VNetCNIPluginsURL $global:VNetCNIPluginsURL + + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` + "`" + ` + -KubeDnsSearchPath $global:KubeDnsSearchPath ` + "`" + ` + -KubeClusterCIDR $global:KubeClusterCIDR ` + "`" + ` + -KubeServiceCIDR $global:KubeServiceCIDR ` + "`" + ` + -VNetCIDR $global:VNetCIDR ` + "`" + ` + -IsDualStackEnabled $global:IsDualStackEnabled ` + "`" + ` + -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + + if ($TargetEnvironment -ieq "AzureStackCloud") { + GenerateAzureStackCNIConfig ` + "`" + ` + -TenantId $global:TenantId ` + "`" + ` + -SubscriptionId $global:SubscriptionId ` + "`" + ` + -ResourceGroup $global:ResourceGroup ` + "`" + ` + -AADClientId $AADClientId ` + "`" + ` + -KubeDir $global:KubeDir ` + "`" + ` + -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` + "`" + ` + -NetworkAPIVersion $NetworkAPIVersion ` + "`" + ` + -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) ` + "`" + ` + -IdentitySystem "{{ GetIdentitySystem }}" + } + + New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled + + Install-KubernetesServices ` + "`" + ` + -KubeDir $global:KubeDir + + Set-Explorer + Adjust-PageFileSize + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" + PREPROVISION_EXTENSION + Update-ServiceFailureActions + Adjust-DynamicPortRange + Register-LogsCleanupScriptTask + Register-NodeResetScriptTask + Update-DefenderPreferences + + $windowsVersion = Get-WindowsVersion + if ($windowsVersion -ne "1809") { + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" + try { + . C:\k\windowssecuretls.ps1 + Enable-SecureTls + } + catch { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS -ErrorMessage $_ + } + } + + Enable-FIPSMode -FipsEnabled $fipsEnabled + if ($global:WindowsGmsaPackageUrl) { + Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl + } + + Check-APIServerConnectivity -MasterIP $MasterIP + + if ($global:WindowsCalicoPackageURL) { + Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp + } + + Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL + + if (Test-Path $CacheDir) + { + Write-Log "Removing aks cache directory" + Remove-Item $CacheDir -Recurse -Force + } + + if ($global:TLSBootstrapToken) { + Write-Log "Removing temporary kube config" + $kubeConfigFile = [io.path]::Combine($KubeDir, "config") + Remove-Item $kubeConfigFile + } + + Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes + + if ($global:RebootNeeded) { + Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot" + Postpone-RestartComputer + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" + Start-ScheduledTask -TaskName "k8s-restart-job" + + $timeout = 180 ## seconds + $timer = [Diagnostics.Stopwatch]::StartNew() + while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') { + # The task ` + "`" + `k8s-restart-job` + "`" + ` needs ~8 seconds. + if ($timer.Elapsed.TotalSeconds -gt $timeout) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds" + } + + Write-Log -Message "Waiting on NodeResetScriptTask..." + Start-Sleep -Seconds 3 + } + $timer.Stop() + Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" + } +} +catch +{ + # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block + # Ideally all exceptions will be handled and no exception will be thrown. + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_ +} +finally +{ + # Generate CSE result so it can be returned as the CSE response in csecmd.ps1 + $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date)) + Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode" + # $CSEResultFilePath is used to avoid running CSE multiple times + Set-Content -Path $CSEResultFilePath -Value $global:ExitCode -Force + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Please not use Write-Log or Logs-To-Events after Stop-Transcript + Stop-Transcript + + # Remove the parameters in the log file to avoid leaking secrets + $logs=Get-Content $LogFile | Where-Object {$_ -notmatch "^Host Application: "} + $logs | Set-Content $LogFile + + Upload-GuestVMLogs -ExitCode $global:ExitCode + if ($global:ExitCode -ne 0) { + # $JsonString = "ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" + # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe. + $errorMessageLength = "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length + $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) + throw "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|" + } +} `) func windowsKuberneteswindowssetupPs1Bytes() ([]byte, error) { @@ -8910,466 +8911,466 @@ func windowsSendlogsPs1() (*asset, error) { return a, nil } -var _windowsWindowscsehelperPs1 = []byte(`# This script is used to define basic util functions -# It is better to define functions in the scripts under staging/cse/windows. - -# Define all exit codes in Windows CSE -# It must match `+"`"+`[A-Z_]+`+"`"+` -$global:WINDOWS_CSE_SUCCESS=0 -$global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY=2 -$global:WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE=3 -$global:WINDOWS_CSE_ERROR_FILE_NOT_EXIST=4 -$global:WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY=5 -$global:WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST=6 -$global:WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX=7 -$global:WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM=8 -$global:WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST=9 -$global:WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST=10 -$global:WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST=11 -$global:WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST=12 -$global:WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED=13 -$global:WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING=14 -$global:WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED=15 -$global:WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED=16 -$global:WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG=17 -$global:WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER=18 -$global:WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES=19 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES=20 -$global:WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES=21 -$global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS=22 -$global:WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE=23 -$global:WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE=24 -$global:WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION=25 -$global:WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES=26 -$global:WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS=27 -$global:WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS=28 -$global:WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP=29 -$global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER=30 -$global:WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS=31 -$global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK=32 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE=33 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE=34 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE=35 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE=36 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE=37 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE=38 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE=39 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE=40 -$global:WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE=41 -$global:WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER=42 -$global:WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE=43 -$global:WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE=44 -$global:WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST=45 -$global:WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE=46 -$global:WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE=47 -$global:WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE=48 -$global:WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN=49 # Return this error code in csecmd.ps1 when C:\AzureData\CustomData.bin does not exist -$global:WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG=50 # Return this error code in csecmd.ps1 when C:\AzureData\CSEResult.log does not exist -$global:WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS=51 -$global:WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE=52 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED=53 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT=54 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED=55 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET=56 -$global:WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND=57 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE=58 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE=59 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION=60 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61 -$global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62 -$global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63 -$global:WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION=64 - -# Please add new error code for downloading new packages in RP code too -$global:ErrorCodeNames = @( - "WINDOWS_CSE_SUCCESS", - "WINDOWS_CSE_ERROR_UNKNOWN", - "WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY", - "WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE", - "WINDOWS_CSE_ERROR_FILE_NOT_EXIST", - "WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY", - "WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST", - "WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX", - "WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM", - "WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST", - "WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST", - "WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST", - "WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST", - "WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED", - "WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING", - "WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED", - "WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED", - "WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG", - "WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER", - "WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES", - "WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES", - "WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES", - "WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS", - "WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE", - "WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE", - "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION", - "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES", - "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS", - "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS", - "WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP", - "WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER", - "WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS", - "WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK", - "WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE", - "WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE", - "WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE", - "WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER", - "WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE", - "WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE", - "WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST", - "WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE", - "WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE", - "WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE", - "WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN", - "WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG", - "WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS", - "WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET", - "WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE", - "WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG", - "WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP", - "WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION" -) - -# NOTE: KubernetesVersion does not contain "v" -$global:MinimalKubernetesVersionWithLatestContainerd = "1.28.0" # Will change it to the correct version when we support new Windows containerd version -# DEPRECATED: The contianerd package url will be set in AKS RP code. We will remove the following variables in the future. -$global:StableContainerdPackage = "v1.6.21-azure.1/binaries/containerd-v1.6.21-azure.1-windows-amd64.tar.gz" -# The latest containerd version -$global:LatestContainerdPackage = "v1.7.9-azure.1/binaries/containerd-v1.7.9-azure.1-windows-amd64.tar.gz" - -$global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" -$global:TaskName = "" -$global:TaskTimeStamp = "" - -# This filter removes null characters (\0) which are captured in nssm.exe output when logged through powershell -filter RemoveNulls { $_ -replace '\0', '' } - -filter Timestamp { "$(Get-Date -Format o): $_" } - -function Write-Log($message) { - $msg = $message | Timestamp - Write-Output $msg -} - -function DownloadFileOverHttp { - Param( - [Parameter(Mandatory = $true)][string] - $Url, - [Parameter(Mandatory = $true)][string] - $DestinationPath, - [Parameter(Mandatory = $true)][int] - $ExitCode - ) - - # First check to see if a file with the same name is already cached on the VHD - $fileName = [IO.Path]::GetFileName($Url) - - $search = @() - if (Test-Path $global:CacheDir) { - $search = [IO.Directory]::GetFiles($global:CacheDir, $fileName, [IO.SearchOption]::AllDirectories) - } - - if ($search.Count -ne 0) { - Write-Log "Using cached version of $fileName - Copying file from $($search[0]) to $DestinationPath" - Copy-Item -Path $search[0] -Destination $DestinationPath -Force - } - else { - $secureProtocols = @() - $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) - - foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) { - if ($insecureProtocols -notcontains $protocol) { - $secureProtocols += $protocol - } - } - [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols - - $oldProgressPreference = $ProgressPreference - $ProgressPreference = 'SilentlyContinue' - - $downloadTimer = [System.Diagnostics.Stopwatch]::StartNew() - try { - $args = @{Uri=$Url; Method="Get"; OutFile=$DestinationPath} - Retry-Command -Command "Invoke-RestMethod" -Args $args -Retries 5 -RetryDelaySeconds 10 - } catch { - Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Failed in downloading $Url. Error: $_" - } - $downloadTimer.Stop() - - if ($global:AppInsightsClient -ne $null) { - $event = New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" - $event.Name = "FileDownload" - $event.Properties["FileName"] = $fileName - $event.Metrics["DurationMs"] = $downloadTimer.ElapsedMilliseconds - $global:AppInsightsClient.TrackEvent($event) - } - - $ProgressPreference = $oldProgressPreference - Write-Log "Downloaded file $Url to $DestinationPath" - } -} - -function Set-ExitCode -{ - Param( - [Parameter(Mandatory=$true)][int] - $ExitCode, - [Parameter(Mandatory=$true)][string] - $ErrorMessage - ) - Write-Log "Set ExitCode to $ExitCode and exit. Error: $ErrorMessage" - $global:ExitCode=$ExitCode - # we use | as the separator as a workaround since " or ' do not work as expected per the testings - $global:ErrorMessage=($ErrorMessage -replace '\|', '%7C') - exit $ExitCode -} - -function Postpone-RestartComputer -{ - Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" - $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command `+"`"+`"Restart-Computer -Force`+"`"+`"" - $principal = New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest - # trigger this task once - $trigger = New-JobTrigger -At (Get-Date).AddSeconds(15).DateTime -Once - $definition = New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "Restart computer after provisioning the VM" - Register-ScheduledTask -TaskName "restart-computer" -InputObject $definition - Write-Log "Created an one-time task to restart the VM" -} - -function Create-Directory -{ - Param( - [Parameter(Mandatory=$true)][string] - $FullPath, - [Parameter(Mandatory=$false)][string] - $DirectoryUsage = "general purpose" - ) - - if (-Not (Test-Path $FullPath)) { - Write-Log "Create directory $FullPath for $DirectoryUsage" - New-Item -ItemType Directory -Path $FullPath > $null - } else { - Write-Log "Directory $FullPath for $DirectoryUsage exists" - } -} - -# https://stackoverflow.com/a/34559554/697126 -function New-TemporaryDirectory { - $parent = [System.IO.Path]::GetTempPath() - [string] $name = [System.Guid]::NewGuid() - New-Item -ItemType Directory -Path (Join-Path $parent $name) -} - -function Retry-Command { - Param( - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] - $Command, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][hashtable] - $Args, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] - $Retries, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] - $RetryDelaySeconds - ) - - for ($i = 0; ; ) { - try { - # Do not log Args since Args may contain sensitive data - Write-Log "Retry $i : $command" - return & $Command @Args - } - catch { - $i++ - if ($i -ge $Retries) { - throw $_ - } - Start-Sleep $RetryDelaySeconds - } - } -} - -function Invoke-Executable { - Param( - [Parameter(Mandatory=$true)][string] - $Executable, - [Parameter(Mandatory=$true)][string[]] - $ArgList, - [Parameter(Mandatory=$true)][int] - $ExitCode, - [int[]] - $AllowedExitCodes = @(0), - [int] - $Retries = 0, - [int] - $RetryDelaySeconds = 1 - ) - - for ($i = 0; $i -le $Retries; $i++) { - Write-Log "$i - Running $Executable $ArgList ..." - & $Executable $ArgList - if ($LASTEXITCODE -notin $AllowedExitCodes) { - Write-Log "$Executable returned unsuccessfully with exit code $LASTEXITCODE" - Start-Sleep -Seconds $RetryDelaySeconds - continue - } - else { - Write-Log "$Executable returned successfully" - return - } - } - - Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Exhausted retries for $Executable $ArgList" -} - -function Assert-FileExists { - Param( - [Parameter(Mandatory = $true)][string] - $Filename, - [Parameter(Mandatory = $true)][int] - $ExitCode - ) - - if (-Not (Test-Path $Filename)) { - Set-ExitCode -ExitCode $ExitCode -ErrorMessage "$Filename does not exist" - } -} - -function Get-WindowsBuildNumber { - return (Get-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion").CurrentBuild -} - -function Get-WindowsVersion { - $buildNumber = Get-WindowsBuildNumber - switch ($buildNumber) { - "17763" { return "1809" } - "20348" { return "ltsc2022" } - "25398" { return "23H2" } - {$_ -ge "25399" -and $_ -le "30397"} { return "test2025" } - Default { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" - } - } -} - -function Get-WindowsPauseVersion { - $buildNumber = Get-WindowsBuildNumber - switch ($buildNumber) { - "17763" { return "1809" } - "20348" { return "ltsc2022" } - "25398" { return "ltsc2022" } - {$_ -ge "25399" -and $_ -le "30397"} { return "ltsc2022" } - Default { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" - } - } -} - -function Install-Containerd-Based-On-Kubernetes-Version { - Param( - [Parameter(Mandatory = $true)][string] - $ContainerdUrl, - [Parameter(Mandatory = $true)][string] - $CNIBinDir, - [Parameter(Mandatory = $true)][string] - $CNIConfDir, - [Parameter(Mandatory = $true)][string] - $KubeDir, - [Parameter(Mandatory = $true)][string] - $KubernetesVersion - ) - - Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion" - - # In the past, $global:ContainerdUrl is a full URL to download Windows containerd package. - # Example: "https://acs-mirror.azureedge.net/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" - # To support multiple containerd versions, we only set the endpoint in $global:ContainerdUrl. - # Example: "https://acs-mirror.azureedge.net/containerd/windows/" - # We only set containerd package based on kubernetes version when $global:ContainerdUrl ends with "/" so we support: - # 1. Current behavior to set the full URL - # 2. Setting containerd package in toggle for test purpose or hotfix - if ($ContainerdUrl.EndsWith("/")) { - Write-Log "ContainerdURL is $ContainerdUrl" - $containerdPackage=$global:StableContainerdPackage - if (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) { - $containerdPackage=$global:LatestContainerdPackage - Write-Log "Kubernetes version $KubernetesVersion is greater than or equal to $global:MinimalKubernetesVersionWithLatestContainerd so the latest containerd version $containerdPackage is used" - } else { - Write-Log "Kubernetes version $KubernetesVersion is less than $global:MinimalKubernetesVersionWithLatestContainerd so the stable containerd version $containerdPackage is used" - } - $ContainerdUrl = $ContainerdUrl + $containerdPackage - } - Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" - Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir -} - -function Logs-To-Event { - Param( - [Parameter(Mandatory = $true)][string] - $TaskName, - [Parameter(Mandatory = $true)][string] - $TaskMessage - ) - $eventLevel="Informational" - if ($global:ExitCode -ne 0) { - $eventLevel="Error" - } - - $eventsFileName=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - $currentTime=$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff") - - $lastTaskName = "" - $lastTaskDuration = 0 - if ($global:TaskTimeStamp -ne "") { - $lastTaskName = $global:TaskName - $lastTaskDuration = $(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) - } - - $global:TaskName = $TaskName - $global:TaskTimeStamp = $currentTime - - Write-Log "$global:TaskName - $TaskMessage" - $TaskMessage = (echo $TaskMessage | ConvertTo-Json) - $messageJson = @" - { - "HostName": "$env:computername", - "LastTaskName": "$lastTaskName", - "LastTaskDuration": "$lastTaskDuration", - "CurrentTaskMessage": $TaskMessage - } -"@ - $messageJson = (echo $messageJson | ConvertTo-Json) - - $jsonString = @" - { - "Timestamp": "$global:TaskTimeStamp", - "OperationId": "$global:OperationId", - "Version": "1.10", - "TaskName": "$global:TaskName", - "EventLevel": "$eventLevel", - "Message": $messageJson - } -"@ - echo $jsonString | Set-Content ${global:EventsLoggingDir}${eventsFileName}.json +var _windowsWindowscsehelperPs1 = []byte(`# This script is used to define basic util functions +# It is better to define functions in the scripts under staging/cse/windows. + +# Define all exit codes in Windows CSE +# It must match ` + "`" + `[A-Z_]+` + "`" + ` +$global:WINDOWS_CSE_SUCCESS=0 +$global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY=2 +$global:WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE=3 +$global:WINDOWS_CSE_ERROR_FILE_NOT_EXIST=4 +$global:WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY=5 +$global:WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST=6 +$global:WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX=7 +$global:WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM=8 +$global:WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST=9 +$global:WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST=10 +$global:WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST=11 +$global:WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST=12 +$global:WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED=13 +$global:WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING=14 +$global:WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED=15 +$global:WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED=16 +$global:WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG=17 +$global:WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER=18 +$global:WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES=19 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES=20 +$global:WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES=21 +$global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS=22 +$global:WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE=23 +$global:WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE=24 +$global:WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION=25 +$global:WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES=26 +$global:WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS=27 +$global:WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS=28 +$global:WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP=29 +$global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER=30 +$global:WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS=31 +$global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK=32 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE=33 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE=34 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE=35 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE=36 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE=37 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE=38 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE=39 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE=40 +$global:WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE=41 +$global:WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER=42 +$global:WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE=43 +$global:WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE=44 +$global:WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST=45 +$global:WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE=46 +$global:WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE=47 +$global:WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE=48 +$global:WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN=49 # Return this error code in csecmd.ps1 when C:\AzureData\CustomData.bin does not exist +$global:WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG=50 # Return this error code in csecmd.ps1 when C:\AzureData\CSEResult.log does not exist +$global:WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS=51 +$global:WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE=52 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED=53 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT=54 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED=55 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET=56 +$global:WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND=57 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE=58 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE=59 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION=60 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61 +$global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62 +$global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63 +$global:WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION=64 + +# Please add new error code for downloading new packages in RP code too +$global:ErrorCodeNames = @( + "WINDOWS_CSE_SUCCESS", + "WINDOWS_CSE_ERROR_UNKNOWN", + "WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY", + "WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE", + "WINDOWS_CSE_ERROR_FILE_NOT_EXIST", + "WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY", + "WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST", + "WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX", + "WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM", + "WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST", + "WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST", + "WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST", + "WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST", + "WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED", + "WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING", + "WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED", + "WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED", + "WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG", + "WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER", + "WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES", + "WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES", + "WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES", + "WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS", + "WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE", + "WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE", + "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION", + "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES", + "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS", + "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS", + "WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP", + "WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER", + "WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS", + "WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK", + "WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE", + "WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE", + "WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE", + "WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER", + "WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE", + "WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE", + "WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST", + "WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE", + "WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE", + "WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE", + "WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN", + "WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG", + "WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS", + "WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET", + "WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE", + "WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG", + "WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP", + "WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION" +) + +# NOTE: KubernetesVersion does not contain "v" +$global:MinimalKubernetesVersionWithLatestContainerd = "1.28.0" # Will change it to the correct version when we support new Windows containerd version +# DEPRECATED: The contianerd package url will be set in AKS RP code. We will remove the following variables in the future. +$global:StableContainerdPackage = "v1.6.21-azure.1/binaries/containerd-v1.6.21-azure.1-windows-amd64.tar.gz" +# The latest containerd version +$global:LatestContainerdPackage = "v1.7.9-azure.1/binaries/containerd-v1.7.9-azure.1-windows-amd64.tar.gz" + +$global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" +$global:TaskName = "" +$global:TaskTimeStamp = "" + +# This filter removes null characters (\0) which are captured in nssm.exe output when logged through powershell +filter RemoveNulls { $_ -replace '\0', '' } + +filter Timestamp { "$(Get-Date -Format o): $_" } + +function Write-Log($message) { + $msg = $message | Timestamp + Write-Output $msg +} + +function DownloadFileOverHttp { + Param( + [Parameter(Mandatory = $true)][string] + $Url, + [Parameter(Mandatory = $true)][string] + $DestinationPath, + [Parameter(Mandatory = $true)][int] + $ExitCode + ) + + # First check to see if a file with the same name is already cached on the VHD + $fileName = [IO.Path]::GetFileName($Url) + + $search = @() + if (Test-Path $global:CacheDir) { + $search = [IO.Directory]::GetFiles($global:CacheDir, $fileName, [IO.SearchOption]::AllDirectories) + } + + if ($search.Count -ne 0) { + Write-Log "Using cached version of $fileName - Copying file from $($search[0]) to $DestinationPath" + Copy-Item -Path $search[0] -Destination $DestinationPath -Force + } + else { + $secureProtocols = @() + $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) + + foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) { + if ($insecureProtocols -notcontains $protocol) { + $secureProtocols += $protocol + } + } + [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols + + $oldProgressPreference = $ProgressPreference + $ProgressPreference = 'SilentlyContinue' + + $downloadTimer = [System.Diagnostics.Stopwatch]::StartNew() + try { + $args = @{Uri=$Url; Method="Get"; OutFile=$DestinationPath} + Retry-Command -Command "Invoke-RestMethod" -Args $args -Retries 5 -RetryDelaySeconds 10 + } catch { + Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Failed in downloading $Url. Error: $_" + } + $downloadTimer.Stop() + + if ($global:AppInsightsClient -ne $null) { + $event = New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" + $event.Name = "FileDownload" + $event.Properties["FileName"] = $fileName + $event.Metrics["DurationMs"] = $downloadTimer.ElapsedMilliseconds + $global:AppInsightsClient.TrackEvent($event) + } + + $ProgressPreference = $oldProgressPreference + Write-Log "Downloaded file $Url to $DestinationPath" + } +} + +function Set-ExitCode +{ + Param( + [Parameter(Mandatory=$true)][int] + $ExitCode, + [Parameter(Mandatory=$true)][string] + $ErrorMessage + ) + Write-Log "Set ExitCode to $ExitCode and exit. Error: $ErrorMessage" + $global:ExitCode=$ExitCode + # we use | as the separator as a workaround since " or ' do not work as expected per the testings + $global:ErrorMessage=($ErrorMessage -replace '\|', '%7C') + exit $ExitCode +} + +function Postpone-RestartComputer +{ + Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" + $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command ` + "`" + `"Restart-Computer -Force` + "`" + `"" + $principal = New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest + # trigger this task once + $trigger = New-JobTrigger -At (Get-Date).AddSeconds(15).DateTime -Once + $definition = New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "Restart computer after provisioning the VM" + Register-ScheduledTask -TaskName "restart-computer" -InputObject $definition + Write-Log "Created an one-time task to restart the VM" +} + +function Create-Directory +{ + Param( + [Parameter(Mandatory=$true)][string] + $FullPath, + [Parameter(Mandatory=$false)][string] + $DirectoryUsage = "general purpose" + ) + + if (-Not (Test-Path $FullPath)) { + Write-Log "Create directory $FullPath for $DirectoryUsage" + New-Item -ItemType Directory -Path $FullPath > $null + } else { + Write-Log "Directory $FullPath for $DirectoryUsage exists" + } +} + +# https://stackoverflow.com/a/34559554/697126 +function New-TemporaryDirectory { + $parent = [System.IO.Path]::GetTempPath() + [string] $name = [System.Guid]::NewGuid() + New-Item -ItemType Directory -Path (Join-Path $parent $name) +} + +function Retry-Command { + Param( + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] + $Command, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][hashtable] + $Args, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] + $Retries, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] + $RetryDelaySeconds + ) + + for ($i = 0; ; ) { + try { + # Do not log Args since Args may contain sensitive data + Write-Log "Retry $i : $command" + return & $Command @Args + } + catch { + $i++ + if ($i -ge $Retries) { + throw $_ + } + Start-Sleep $RetryDelaySeconds + } + } +} + +function Invoke-Executable { + Param( + [Parameter(Mandatory=$true)][string] + $Executable, + [Parameter(Mandatory=$true)][string[]] + $ArgList, + [Parameter(Mandatory=$true)][int] + $ExitCode, + [int[]] + $AllowedExitCodes = @(0), + [int] + $Retries = 0, + [int] + $RetryDelaySeconds = 1 + ) + + for ($i = 0; $i -le $Retries; $i++) { + Write-Log "$i - Running $Executable $ArgList ..." + & $Executable $ArgList + if ($LASTEXITCODE -notin $AllowedExitCodes) { + Write-Log "$Executable returned unsuccessfully with exit code $LASTEXITCODE" + Start-Sleep -Seconds $RetryDelaySeconds + continue + } + else { + Write-Log "$Executable returned successfully" + return + } + } + + Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Exhausted retries for $Executable $ArgList" +} + +function Assert-FileExists { + Param( + [Parameter(Mandatory = $true)][string] + $Filename, + [Parameter(Mandatory = $true)][int] + $ExitCode + ) + + if (-Not (Test-Path $Filename)) { + Set-ExitCode -ExitCode $ExitCode -ErrorMessage "$Filename does not exist" + } +} + +function Get-WindowsBuildNumber { + return (Get-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion").CurrentBuild +} + +function Get-WindowsVersion { + $buildNumber = Get-WindowsBuildNumber + switch ($buildNumber) { + "17763" { return "1809" } + "20348" { return "ltsc2022" } + "25398" { return "23H2" } + {$_ -ge "25399" -and $_ -le "30397"} { return "test2025" } + Default { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" + } + } +} + +function Get-WindowsPauseVersion { + $buildNumber = Get-WindowsBuildNumber + switch ($buildNumber) { + "17763" { return "1809" } + "20348" { return "ltsc2022" } + "25398" { return "ltsc2022" } + {$_ -ge "25399" -and $_ -le "30397"} { return "ltsc2022" } + Default { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" + } + } +} + +function Install-Containerd-Based-On-Kubernetes-Version { + Param( + [Parameter(Mandatory = $true)][string] + $ContainerdUrl, + [Parameter(Mandatory = $true)][string] + $CNIBinDir, + [Parameter(Mandatory = $true)][string] + $CNIConfDir, + [Parameter(Mandatory = $true)][string] + $KubeDir, + [Parameter(Mandatory = $true)][string] + $KubernetesVersion + ) + + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion" + + # In the past, $global:ContainerdUrl is a full URL to download Windows containerd package. + # Example: "https://acs-mirror.azureedge.net/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" + # To support multiple containerd versions, we only set the endpoint in $global:ContainerdUrl. + # Example: "https://acs-mirror.azureedge.net/containerd/windows/" + # We only set containerd package based on kubernetes version when $global:ContainerdUrl ends with "/" so we support: + # 1. Current behavior to set the full URL + # 2. Setting containerd package in toggle for test purpose or hotfix + if ($ContainerdUrl.EndsWith("/")) { + Write-Log "ContainerdURL is $ContainerdUrl" + $containerdPackage=$global:StableContainerdPackage + if (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) { + $containerdPackage=$global:LatestContainerdPackage + Write-Log "Kubernetes version $KubernetesVersion is greater than or equal to $global:MinimalKubernetesVersionWithLatestContainerd so the latest containerd version $containerdPackage is used" + } else { + Write-Log "Kubernetes version $KubernetesVersion is less than $global:MinimalKubernetesVersionWithLatestContainerd so the stable containerd version $containerdPackage is used" + } + $ContainerdUrl = $ContainerdUrl + $containerdPackage + } + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" + Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir +} + +function Logs-To-Event { + Param( + [Parameter(Mandatory = $true)][string] + $TaskName, + [Parameter(Mandatory = $true)][string] + $TaskMessage + ) + $eventLevel="Informational" + if ($global:ExitCode -ne 0) { + $eventLevel="Error" + } + + $eventsFileName=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $currentTime=$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff") + + $lastTaskName = "" + $lastTaskDuration = 0 + if ($global:TaskTimeStamp -ne "") { + $lastTaskName = $global:TaskName + $lastTaskDuration = $(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) + } + + $global:TaskName = $TaskName + $global:TaskTimeStamp = $currentTime + + Write-Log "$global:TaskName - $TaskMessage" + $TaskMessage = (echo $TaskMessage | ConvertTo-Json) + $messageJson = @" + { + "HostName": "$env:computername", + "LastTaskName": "$lastTaskName", + "LastTaskDuration": "$lastTaskDuration", + "CurrentTaskMessage": $TaskMessage + } +"@ + $messageJson = (echo $messageJson | ConvertTo-Json) + + $jsonString = @" + { + "Timestamp": "$global:TaskTimeStamp", + "OperationId": "$global:OperationId", + "Version": "1.10", + "TaskName": "$global:TaskName", + "EventLevel": "$eventLevel", + "Message": $messageJson + } +"@ + echo $jsonString | Set-Content ${global:EventsLoggingDir}${eventsFileName}.json }`) func windowsWindowscsehelperPs1Bytes() ([]byte, error) { @@ -9549,11 +9550,13 @@ var _bindata = map[string]func() (*asset, error){ // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the // following hierarchy: -// data/ -// foo.txt -// img/ -// a.png -// b.png +// +// data/ +// foo.txt +// img/ +// a.png +// b.png +// // then AssetDir("data") would return []string{"foo.txt", "img"} // AssetDir("data/img") would return []string{"a.png", "b.png"} // AssetDir("foo.txt") and AssetDir("notexist") would return an error From 3385a40a1a05d0c2ec7b452725aa91069ec7f0d2 Mon Sep 17 00:00:00 2001 From: Artun Duman Date: Tue, 9 Apr 2024 16:34:59 -0700 Subject: [PATCH 3/3] revert fmt for templates generated --- pkg/templates/templates_generated.go | 1979 +++++++++++++------------- 1 file changed, 988 insertions(+), 991 deletions(-) diff --git a/pkg/templates/templates_generated.go b/pkg/templates/templates_generated.go index bbd294b4563..4215ab52247 100644 --- a/pkg/templates/templates_generated.go +++ b/pkg/templates/templates_generated.go @@ -114,7 +114,6 @@ import ( "strings" "time" ) - type asset struct { bytes []byte info os.FileInfo @@ -3151,7 +3150,7 @@ ERR_AZURE_STACK_GET_ARM_TOKEN=120 # Error generating a token to use with Azure R ERR_AZURE_STACK_GET_NETWORK_CONFIGURATION=121 # Error fetching the network configuration for the node ERR_AZURE_STACK_GET_SUBNET_PREFIX=122 # Error fetching the subnet address prefix for a subnet ID -# Error code 124 is returned when a ` + "`" + `timeout` + "`" + ` command times out, and --preserve-status is not specified: https://man7.org/linux/man-pages/man1/timeout.1.html +# Error code 124 is returned when a `+"`"+`timeout`+"`"+` command times out, and --preserve-status is not specified: https://man7.org/linux/man-pages/man1/timeout.1.html ERR_VHD_BUILD_ERROR=125 # Reserved for VHD CI exit conditions ERR_SWAP_CREATE_FAIL=130 # Error allocating swap file @@ -3187,7 +3186,7 @@ MARINER_OS_NAME="MARINER" KUBECTL=/usr/local/bin/kubectl DOCKER=/usr/bin/docker # this will be empty during VHD build -# but vhd build runs with ` + "`" + `set -o nounset` + "`" + ` +# but vhd build runs with `+"`"+`set -o nounset`+"`"+` # so needs a default value # prefer empty string to avoid potential "it works but did something weird" scenarios export GPU_DV="${GPU_DRIVER_VERSION:=}" @@ -3796,7 +3795,7 @@ installKubeletKubectlAndKubeProxy() { # remove the kubelet and kubectl binaries to make sure the only binary left is from the CUSTOM_KUBE_BINARY_DOWNLOAD_URL rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* - # NOTE(mainred): we expect kubelet binary to be under ` + "`" + `kubernetes/node/bin` + "`" + `. This suits the current setting of + # NOTE(mainred): we expect kubelet binary to be under `+"`"+`kubernetes/node/bin`+"`"+`. This suits the current setting of # kube binaries used by AKS and Kubernetes upstream. # TODO(mainred): let's see if necessary to auto-detect the path of kubelet logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL} false @@ -4169,8 +4168,8 @@ EOF if [[ "${MIG_NODE}" == "true" ]]; then # A100 GPU has a bit in the physical card (infoROM) to enable mig mode. # Changing this bit in either direction requires a VM reboot on Azure (hypervisor/plaform stuff). - # Commands such as ` + "`" + `nvidia-smi --gpu-reset` + "`" + ` may succeed, - # while commands such as ` + "`" + `nvidia-smi -q` + "`" + ` will show mismatched current/pending mig mode. + # Commands such as `+"`"+`nvidia-smi --gpu-reset`+"`"+` may succeed, + # while commands such as `+"`"+`nvidia-smi -q`+"`"+` will show mismatched current/pending mig mode. # this will not be required per nvidia for next gen H100. REBOOTREQUIRED=true @@ -6185,7 +6184,7 @@ var _linuxCloudInitArtifactsPamDCommonPassword = []byte(`# # The "sha512" option enables salted SHA512 passwords. Without this option, # the default is Unix crypt. Prior releases used the option "md5". # -# The "obscure" option replaces the old ` + "`" + `OBSCURE_CHECKS_ENAB' option in +# The "obscure" option replaces the old `+"`"+`OBSCURE_CHECKS_ENAB' option in # login.defs. # # See the pam_unix manpage for other options. @@ -6229,19 +6228,19 @@ func linuxCloudInitArtifactsPamDCommonPassword() (*asset, error) { } var _linuxCloudInitArtifactsPamDSu = []byte(`# -# The PAM configuration file for the Shadow ` + "`" + `su' service +# The PAM configuration file for the Shadow `+"`"+`su' service # # This allows root to su without passwords (normal operation) auth sufficient pam_rootok.so # Uncomment this to force users to be a member of group root -# before they can use ` + "`" + `su'. You can also add "group=foo" +# before they can use `+"`"+`su'. You can also add "group=foo" # to the end of this line if you want to use a group other # than the default "root" (but this may have side effect of # denying "root" user, unless she's a member of "foo" or explicitly # permitted earlier by e.g. "sufficient pam_rootok.so"). -# (Replaces the ` + "`" + `SU_WHEEL_ONLY' option from login.defs) +# (Replaces the `+"`"+`SU_WHEEL_ONLY' option from login.defs) # 5.6 Ensure access to the su command is restricted auth required pam_wheel.so use_uid @@ -6256,7 +6255,7 @@ auth required pam_wheel.so use_uid # Uncomment and edit /etc/security/time.conf if you need to set # time restrainst on su usage. -# (Replaces the ` + "`" + `PORTTIME_CHECKS_ENAB' option from login.defs +# (Replaces the `+"`"+`PORTTIME_CHECKS_ENAB' option from login.defs # as well as /etc/porttime) # account requisite pam_time.so @@ -8324,519 +8323,519 @@ func windowsCsecmdPs1() (*asset, error) { return a, nil } -var _windowsKuberneteswindowssetupPs1 = []byte(`<# - .SYNOPSIS - Provisions VM as a Kubernetes agent. - - .DESCRIPTION - Provisions VM as a Kubernetes agent. - - The parameters passed in are required, and will vary per-deployment. - - Notes on modifying this file: - - This file extension is PS1, but it is actually used as a template from pkg/engine/template_generator.go - - All of the lines that have braces in them will be modified. Please do not change them here, change them in the Go sources - - Single quotes are forbidden, they are reserved to delineate the different members for the ARM template concat() call - - windowscsehelper.ps1 contains basic util functions. It will be compressed to a zip file and then be converted to base64 encoding - string and stored in $zippedFiles. Reason: This script is a template and has some limitations. - - All other scripts will be packaged and published in a single package. It will be downloaded in provisioning VM. - Reason: CustomData has length limitation 87380. - - ProvisioningScriptsPackage contains scripts to start kubelet, kubeproxy, etc. The source is https://github.com/Azure/aks-engine/tree/master/staging/provisioning/windows -#> -[CmdletBinding(DefaultParameterSetName="Standard")] -param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AgentKey, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientSecret, # base64 - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $LogFile, - - # C:\AzureData\provision.complete - # MUST keep generating this file when CSE is done and do not change the name - # - It is used to avoid running CSE multiple times - # - Some customers use this file to check if CSE is done - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID -) -# Do not parse the start time from $LogFile to simplify the logic -$StartTime=Get-Date -$global:ExitCode=0 -$global:ErrorMessage="" -Start-Transcript -Path $LogFile -# These globals will not change between nodes in the same cluster, so they are not -# passed as powershell parameters - -## SSH public keys to add to authorized_keys -$global:SSHKeys = @( {{ GetSshPublicKeysPowerShell }} ) - -## Certificates generated by aks-engine -$global:CACertificate = "{{GetParameter "caCertificate"}}" -$global:AgentCertificate = "{{GetParameter "clientCertificate"}}" - -## Download sources provided by aks-engine -$global:KubeBinariesPackageSASURL = "{{GetParameter "kubeBinariesSASURL"}}" -$global:WindowsKubeBinariesURL = "{{GetParameter "windowsKubeBinariesURL"}}" -$global:KubeBinariesVersion = "{{GetParameter "kubeBinariesVersion"}}" -$global:ContainerdUrl = "{{GetParameter "windowsContainerdURL"}}" -$global:ContainerdSdnPluginUrl = "{{GetParameter "windowsSdnPluginURL"}}" - -## Docker Version -$global:DockerVersion = "{{GetParameter "windowsDockerVersion"}}" - -## ContainerD Usage -$global:DefaultContainerdWindowsSandboxIsolation = "{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}" -$global:ContainerdWindowsRuntimeHandlers = "{{GetParameter "containerdWindowsRuntimeHandlers"}}" - -## VM configuration passed by Azure -$global:WindowsTelemetryGUID = "{{GetParameter "windowsTelemetryGUID"}}" -{{if eq GetIdentitySystem "adfs"}} -$global:TenantId = "adfs" -{{else}} -$global:TenantId = "{{GetVariable "tenantID"}}" -{{end}} -$global:SubscriptionId = "{{GetVariable "subscriptionId"}}" -$global:ResourceGroup = "{{GetVariable "resourceGroup"}}" -$global:VmType = "{{GetVariable "vmType"}}" -$global:SubnetName = "{{GetVariable "subnetName"}}" -# NOTE: MasterSubnet is still referenced by ` + "`" + `kubeletstart.ps1` + "`" + ` and ` + "`" + `windowsnodereset.ps1` + "`" + ` -# for case of Kubenet -$global:MasterSubnet = "" -$global:SecurityGroupName = "{{GetVariable "nsgName"}}" -$global:VNetName = "{{GetVariable "virtualNetworkName"}}" -$global:RouteTableName = "{{GetVariable "routeTableName"}}" -$global:PrimaryAvailabilitySetName = "{{GetVariable "primaryAvailabilitySetName"}}" -$global:PrimaryScaleSetName = "{{GetVariable "primaryScaleSetName"}}" - -$global:KubeClusterCIDR = "{{GetParameter "kubeClusterCidr"}}" -$global:KubeServiceCIDR = "{{GetParameter "kubeServiceCidr"}}" -$global:VNetCIDR = "{{GetParameter "vnetCidr"}}" -{{if IsKubernetesVersionGe "1.16.0"}} -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . }}" -{{else}} -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabelsDeprecated . }}" -{{end}} -$global:KubeletConfigArgs = @( {{GetKubeletConfigKeyValsPsh}} ) -$global:KubeproxyConfigArgs = @( {{GetKubeproxyConfigKeyValsPsh}} ) - -$global:KubeproxyFeatureGates = @( {{GetKubeProxyFeatureGatesPsh}} ) - -$global:UseManagedIdentityExtension = "{{GetVariable "useManagedIdentityExtension"}}" -$global:UseInstanceMetadata = "{{GetVariable "useInstanceMetadata"}}" - -$global:LoadBalancerSku = "{{GetVariable "loadBalancerSku"}}" -$global:ExcludeMasterFromStandardLB = "{{GetVariable "excludeMasterFromStandardLB"}}" - -$global:PrivateEgressProxyAddress = "{{GetPrivateEgressProxyAddress}}" - -# Windows defaults, not changed by aks-engine -$global:CacheDir = "c:\akse-cache" -$global:KubeDir = "c:\k" -$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") - -$global:KubeDnsSearchPath = "svc.cluster.local" - -$global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") -$global:NetworkMode = "L2Bridge" -$global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "` + "`" + `$global:NetworkMode.conf") -$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") - - -$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") -$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") -$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") - -# Azure cni configuration -# $global:NetworkPolicy = "{{GetParameter "networkPolicy"}}" # BUG: unused -$global:NetworkPlugin = "{{GetParameter "networkPlugin"}}" -$global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" -$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} -$global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} - -# CSI Proxy settings -$global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}"); -$global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}"; - -# Hosts Config Agent settings -$global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("{{ EnableHostsConfigAgent }}"); - -# These scripts are used by cse -$global:CSEScriptsPackageUrl = "{{GetVariable "windowsCSEScriptsPackageURL" }}"; - -# The windows nvidia gpu driver related url is used by windows cse -$global:GpuDriverURL = "{{GetVariable "windowsGpuDriverURL" }}"; - -# PauseImage -$global:WindowsPauseImageURL = "{{GetVariable "windowsPauseImageURL" }}"; -$global:AlwaysPullWindowsPauseImage = [System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage" }}"); - -# Calico -$global:WindowsCalicoPackageURL = "{{GetVariable "windowsCalicoPackageURL" }}"; - -## GPU install -$global:ConfigGPUDriverIfNeeded = [System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded" }}"); - -# GMSA -$global:WindowsGmsaPackageUrl = "{{GetVariable "windowsGmsaPackageUrl" }}"; - -# TLS Bootstrap Token -$global:TLSBootstrapToken = "{{GetTLSBootstrapTokenForKubeConfig}}" - -# Disable OutBoundNAT in Azure CNI configuration -$global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat" }}"); - -# Base64 representation of ZIP archive -$zippedFiles = "{{ GetKubernetesWindowsAgentFunctions }}" - -$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" -$fipsEnabled = [System.Convert]::ToBoolean("{{ FIPSEnabled }}") - -# HNS remediator -$global:HNSRemediatorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}"); - -# Log generator -$global:LogGeneratorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}"); - -$global:EnableIncreaseDynamicPortRange = $false - -$global:RebootNeeded = $false - -# Extract cse helper script from ZIP -[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) -Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force - -# Dot-source windowscsehelper.ps1 with functions that are called in this script -. c:\AzureData\windows\windowscsehelper.ps1 -# util functions only can be used after this line, for example, Write-Log - -$global:OperationId = New-Guid - -try -{ - Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" - - # Exit early if the script has been executed - if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { - Write-Log "The script has been executed before, will exit without doing anything." - return - } - - # This involes using proxy, log the config before fetching packages - Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'" - # TODO update to use proxy - - $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.40.zip" - Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" - Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" - # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl - # but it is better to set the CSE package version in Windows CSE in AgentBaker - # since most changes in CSE package also need the change in Windows CSE in AgentBaker - # In future, AKS RP only sets the endpoint with the pacakge name, for example, https://acs-mirror.azureedge.net/aks/windows/cse/ - if ($global:CSEScriptsPackageUrl.EndsWith("/")) { - $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage - Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" - } - - # Download CSE function scripts - Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" - $tempfile = 'c:\csescripts.zip' - DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE - Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" -Force - Remove-Item -Path $tempfile -Force - - # Dot-source cse scripts with functions that are called in this script - . c:\AzureData\windows\azurecnifunc.ps1 - . c:\AzureData\windows\calicofunc.ps1 - . c:\AzureData\windows\configfunc.ps1 - . c:\AzureData\windows\containerdfunc.ps1 - . c:\AzureData\windows\kubeletfunc.ps1 - . c:\AzureData\windows\kubernetesfunc.ps1 - . c:\AzureData\windows\nvidiagpudriverfunc.ps1 - - # Install OpenSSH if SSH enabled - $sshEnabled = [System.Convert]::ToBoolean("{{ WindowsSSHEnabled }}") - - if ( $sshEnabled ) { - Install-OpenSSH -SSHKeys $SSHKeys - } - - Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - - Resize-OSDrive - - Initialize-DataDisks - - Initialize-DataDirectories - - Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" - Create-Directory -FullPath "c:\k" - Write-Log "Remove ` + "`" + `"NT AUTHORITY\Authenticated Users` + "`" + `" write permissions on files in c:\k" - icacls.exe "c:\k" /inheritance:r - icacls.exe "c:\k" /grant:r SYSTEM:` + "`" + `(OI` + "`" + `)` + "`" + `(CI` + "`" + `)` + "`" + `(F` + "`" + `) - icacls.exe "c:\k" /grant:r BUILTIN\Administrators:` + "`" + `(OI` + "`" + `)` + "`" + `(CI` + "`" + `)` + "`" + `(F` + "`" + `) - icacls.exe "c:\k" /grant:r BUILTIN\Users:` + "`" + `(OI` + "`" + `)` + "`" + `(CI` + "`" + `)` + "`" + `(RX` + "`" + `) - Write-Log "c:\k permissions: " - icacls.exe "c:\k" - Get-ProvisioningScripts - Get-LogCollectionScripts - - Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - $cniBinPath = $global:AzureCNIBinDir - $cniConfigPath = $global:AzureCNIConfDir - if ($global:NetworkPlugin -eq "kubenet") { - $cniBinPath = $global:CNIPath - $cniConfigPath = $global:CNIConfigPath - } - - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion - - Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - - # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-AzureConfig ` + "`" + ` - -KubeDir $global:KubeDir ` + "`" + ` - -AADClientId $AADClientId ` + "`" + ` - -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` + "`" + ` - -TenantId $global:TenantId ` + "`" + ` - -SubscriptionId $global:SubscriptionId ` + "`" + ` - -ResourceGroup $global:ResourceGroup ` + "`" + ` - -Location $Location ` + "`" + ` - -VmType $global:VmType ` + "`" + ` - -SubnetName $global:SubnetName ` + "`" + ` - -SecurityGroupName $global:SecurityGroupName ` + "`" + ` - -VNetName $global:VNetName ` + "`" + ` - -RouteTableName $global:RouteTableName ` + "`" + ` - -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName ` + "`" + ` - -PrimaryScaleSetName $global:PrimaryScaleSetName ` + "`" + ` - -UseManagedIdentityExtension $global:UseManagedIdentityExtension ` + "`" + ` - -UserAssignedClientID $UserAssignedClientID ` + "`" + ` - -UseInstanceMetadata $global:UseInstanceMetadata ` + "`" + ` - -LoadBalancerSku $global:LoadBalancerSku ` + "`" + ` - -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB ` + "`" + ` - -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud"{{else}}$TargetEnvironment{{end}} - - # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud. - # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - {{if IsAKSCustomCloud}} - $azureStackConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") - $envJSON = "{{ GetBase64EncodedEnvironmentJSON }}" - [io.file]::WriteAllBytes($azureStackConfigFile, [System.Convert]::FromBase64String($envJSON)) - - Get-CACertificates - {{end}} - - Write-CACert -CACertificate $global:CACertificate ` + "`" + ` - -KubeDir $global:KubeDir - - if ($global:EnableCsiProxy) { - New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir - } - - if ($global:TLSBootstrapToken) { - Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` + "`" + ` - -KubeDir $global:KubeDir ` + "`" + ` - -MasterFQDNPrefix $MasterFQDNPrefix ` + "`" + ` - -MasterIP $MasterIP ` + "`" + ` - -TLSBootstrapToken $global:TLSBootstrapToken - - # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled - # This kubeconfig will deleted after calico installation. - # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block - Write-Log "Write temporary kube config" - } else { - Write-Log "Write kube config" - } - - Write-KubeConfig -CACertificate $global:CACertificate ` + "`" + ` - -KubeDir $global:KubeDir ` + "`" + ` - -MasterFQDNPrefix $MasterFQDNPrefix ` + "`" + ` - -MasterIP $MasterIP ` + "`" + ` - -AgentKey $AgentKey ` + "`" + ` - -AgentCertificate $global:AgentCertificate - - if ($global:EnableHostsConfigAgent) { - New-HostsConfigService - } - - Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin" - - # Configure network policy. - Get-HnsPsm1 -HNSModule $global:HNSModule - Import-Module $global:HNSModule - - Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` + "`" + ` - -AzureCNIBinDir $global:AzureCNIBinDir ` + "`" + ` - -VNetCNIPluginsURL $global:VNetCNIPluginsURL - - Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` + "`" + ` - -KubeDnsSearchPath $global:KubeDnsSearchPath ` + "`" + ` - -KubeClusterCIDR $global:KubeClusterCIDR ` + "`" + ` - -KubeServiceCIDR $global:KubeServiceCIDR ` + "`" + ` - -VNetCIDR $global:VNetCIDR ` + "`" + ` - -IsDualStackEnabled $global:IsDualStackEnabled ` + "`" + ` - -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - - if ($TargetEnvironment -ieq "AzureStackCloud") { - GenerateAzureStackCNIConfig ` + "`" + ` - -TenantId $global:TenantId ` + "`" + ` - -SubscriptionId $global:SubscriptionId ` + "`" + ` - -ResourceGroup $global:ResourceGroup ` + "`" + ` - -AADClientId $AADClientId ` + "`" + ` - -KubeDir $global:KubeDir ` + "`" + ` - -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) ` + "`" + ` - -NetworkAPIVersion $NetworkAPIVersion ` + "`" + ` - -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) ` + "`" + ` - -IdentitySystem "{{ GetIdentitySystem }}" - } - - New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - - Install-KubernetesServices ` + "`" + ` - -KubeDir $global:KubeDir - - Set-Explorer - Adjust-PageFileSize - Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - PREPROVISION_EXTENSION - Update-ServiceFailureActions - Adjust-DynamicPortRange - Register-LogsCleanupScriptTask - Register-NodeResetScriptTask - Update-DefenderPreferences - - $windowsVersion = Get-WindowsVersion - if ($windowsVersion -ne "1809") { - Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" - } else { - Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" - try { - . C:\k\windowssecuretls.ps1 - Enable-SecureTls - } - catch { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS -ErrorMessage $_ - } - } - - Enable-FIPSMode -FipsEnabled $fipsEnabled - if ($global:WindowsGmsaPackageUrl) { - Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl - } - - Check-APIServerConnectivity -MasterIP $MasterIP - - if ($global:WindowsCalicoPackageURL) { - Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp - } - - Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - - if (Test-Path $CacheDir) - { - Write-Log "Removing aks cache directory" - Remove-Item $CacheDir -Recurse -Force - } - - if ($global:TLSBootstrapToken) { - Write-Log "Removing temporary kube config" - $kubeConfigFile = [io.path]::Combine($KubeDir, "config") - Remove-Item $kubeConfigFile - } - - Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - if ($global:RebootNeeded) { - Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot" - Postpone-RestartComputer - } else { - Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" - Start-ScheduledTask -TaskName "k8s-restart-job" - - $timeout = 180 ## seconds - $timer = [Diagnostics.Stopwatch]::StartNew() - while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') { - # The task ` + "`" + `k8s-restart-job` + "`" + ` needs ~8 seconds. - if ($timer.Elapsed.TotalSeconds -gt $timeout) { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds" - } - - Write-Log -Message "Waiting on NodeResetScriptTask..." - Start-Sleep -Seconds 3 - } - $timer.Stop() - Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" - } -} -catch -{ - # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block - # Ideally all exceptions will be handled and no exception will be thrown. - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_ -} -finally -{ - # Generate CSE result so it can be returned as the CSE response in csecmd.ps1 - $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date)) - Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode" - # $CSEResultFilePath is used to avoid running CSE multiple times - Set-Content -Path $CSEResultFilePath -Value $global:ExitCode -Force - Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." - # Please not use Write-Log or Logs-To-Events after Stop-Transcript - Stop-Transcript - - # Remove the parameters in the log file to avoid leaking secrets - $logs=Get-Content $LogFile | Where-Object {$_ -notmatch "^Host Application: "} - $logs | Set-Content $LogFile - - Upload-GuestVMLogs -ExitCode $global:ExitCode - if ($global:ExitCode -ne 0) { - # $JsonString = "ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" - # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe. - $errorMessageLength = "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length - $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) - throw "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|" - } -} +var _windowsKuberneteswindowssetupPs1 = []byte(`<# + .SYNOPSIS + Provisions VM as a Kubernetes agent. + + .DESCRIPTION + Provisions VM as a Kubernetes agent. + + The parameters passed in are required, and will vary per-deployment. + + Notes on modifying this file: + - This file extension is PS1, but it is actually used as a template from pkg/engine/template_generator.go + - All of the lines that have braces in them will be modified. Please do not change them here, change them in the Go sources + - Single quotes are forbidden, they are reserved to delineate the different members for the ARM template concat() call + - windowscsehelper.ps1 contains basic util functions. It will be compressed to a zip file and then be converted to base64 encoding + string and stored in $zippedFiles. Reason: This script is a template and has some limitations. + - All other scripts will be packaged and published in a single package. It will be downloaded in provisioning VM. + Reason: CustomData has length limitation 87380. + - ProvisioningScriptsPackage contains scripts to start kubelet, kubeproxy, etc. The source is https://github.com/Azure/aks-engine/tree/master/staging/provisioning/windows +#> +[CmdletBinding(DefaultParameterSetName="Standard")] +param( + [string] + [ValidateNotNullOrEmpty()] + $MasterIP, + + [parameter()] + [ValidateNotNullOrEmpty()] + $KubeDnsServiceIp, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $MasterFQDNPrefix, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $Location, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AgentKey, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AADClientId, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $AADClientSecret, # base64 + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $NetworkAPIVersion, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $TargetEnvironment, + + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $LogFile, + + # C:\AzureData\provision.complete + # MUST keep generating this file when CSE is done and do not change the name + # - It is used to avoid running CSE multiple times + # - Some customers use this file to check if CSE is done + [parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $CSEResultFilePath, + + [string] + $UserAssignedClientID +) +# Do not parse the start time from $LogFile to simplify the logic +$StartTime=Get-Date +$global:ExitCode=0 +$global:ErrorMessage="" +Start-Transcript -Path $LogFile +# These globals will not change between nodes in the same cluster, so they are not +# passed as powershell parameters + +## SSH public keys to add to authorized_keys +$global:SSHKeys = @( {{ GetSshPublicKeysPowerShell }} ) + +## Certificates generated by aks-engine +$global:CACertificate = "{{GetParameter "caCertificate"}}" +$global:AgentCertificate = "{{GetParameter "clientCertificate"}}" + +## Download sources provided by aks-engine +$global:KubeBinariesPackageSASURL = "{{GetParameter "kubeBinariesSASURL"}}" +$global:WindowsKubeBinariesURL = "{{GetParameter "windowsKubeBinariesURL"}}" +$global:KubeBinariesVersion = "{{GetParameter "kubeBinariesVersion"}}" +$global:ContainerdUrl = "{{GetParameter "windowsContainerdURL"}}" +$global:ContainerdSdnPluginUrl = "{{GetParameter "windowsSdnPluginURL"}}" + +## Docker Version +$global:DockerVersion = "{{GetParameter "windowsDockerVersion"}}" + +## ContainerD Usage +$global:DefaultContainerdWindowsSandboxIsolation = "{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}" +$global:ContainerdWindowsRuntimeHandlers = "{{GetParameter "containerdWindowsRuntimeHandlers"}}" + +## VM configuration passed by Azure +$global:WindowsTelemetryGUID = "{{GetParameter "windowsTelemetryGUID"}}" +{{if eq GetIdentitySystem "adfs"}} +$global:TenantId = "adfs" +{{else}} +$global:TenantId = "{{GetVariable "tenantID"}}" +{{end}} +$global:SubscriptionId = "{{GetVariable "subscriptionId"}}" +$global:ResourceGroup = "{{GetVariable "resourceGroup"}}" +$global:VmType = "{{GetVariable "vmType"}}" +$global:SubnetName = "{{GetVariable "subnetName"}}" +# NOTE: MasterSubnet is still referenced by `+"`"+`kubeletstart.ps1`+"`"+` and `+"`"+`windowsnodereset.ps1`+"`"+` +# for case of Kubenet +$global:MasterSubnet = "" +$global:SecurityGroupName = "{{GetVariable "nsgName"}}" +$global:VNetName = "{{GetVariable "virtualNetworkName"}}" +$global:RouteTableName = "{{GetVariable "routeTableName"}}" +$global:PrimaryAvailabilitySetName = "{{GetVariable "primaryAvailabilitySetName"}}" +$global:PrimaryScaleSetName = "{{GetVariable "primaryScaleSetName"}}" + +$global:KubeClusterCIDR = "{{GetParameter "kubeClusterCidr"}}" +$global:KubeServiceCIDR = "{{GetParameter "kubeServiceCidr"}}" +$global:VNetCIDR = "{{GetParameter "vnetCidr"}}" +{{if IsKubernetesVersionGe "1.16.0"}} +$global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . }}" +{{else}} +$global:KubeletNodeLabels = "{{GetAgentKubernetesLabelsDeprecated . }}" +{{end}} +$global:KubeletConfigArgs = @( {{GetKubeletConfigKeyValsPsh}} ) +$global:KubeproxyConfigArgs = @( {{GetKubeproxyConfigKeyValsPsh}} ) + +$global:KubeproxyFeatureGates = @( {{GetKubeProxyFeatureGatesPsh}} ) + +$global:UseManagedIdentityExtension = "{{GetVariable "useManagedIdentityExtension"}}" +$global:UseInstanceMetadata = "{{GetVariable "useInstanceMetadata"}}" + +$global:LoadBalancerSku = "{{GetVariable "loadBalancerSku"}}" +$global:ExcludeMasterFromStandardLB = "{{GetVariable "excludeMasterFromStandardLB"}}" + +$global:PrivateEgressProxyAddress = "{{GetPrivateEgressProxyAddress}}" + +# Windows defaults, not changed by aks-engine +$global:CacheDir = "c:\akse-cache" +$global:KubeDir = "c:\k" +$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.v2.psm1") + +$global:KubeDnsSearchPath = "svc.cluster.local" + +$global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") +$global:NetworkMode = "L2Bridge" +$global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`+"`"+`$global:NetworkMode.conf") +$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") + + +$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") +$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") +$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") + +# Azure cni configuration +# $global:NetworkPolicy = "{{GetParameter "networkPolicy"}}" # BUG: unused +$global:NetworkPlugin = "{{GetParameter "networkPlugin"}}" +$global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" +$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} +$global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} + +# CSI Proxy settings +$global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}"); +$global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}"; + +# Hosts Config Agent settings +$global:EnableHostsConfigAgent = [System.Convert]::ToBoolean("{{ EnableHostsConfigAgent }}"); + +# These scripts are used by cse +$global:CSEScriptsPackageUrl = "{{GetVariable "windowsCSEScriptsPackageURL" }}"; + +# The windows nvidia gpu driver related url is used by windows cse +$global:GpuDriverURL = "{{GetVariable "windowsGpuDriverURL" }}"; + +# PauseImage +$global:WindowsPauseImageURL = "{{GetVariable "windowsPauseImageURL" }}"; +$global:AlwaysPullWindowsPauseImage = [System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage" }}"); + +# Calico +$global:WindowsCalicoPackageURL = "{{GetVariable "windowsCalicoPackageURL" }}"; + +## GPU install +$global:ConfigGPUDriverIfNeeded = [System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded" }}"); + +# GMSA +$global:WindowsGmsaPackageUrl = "{{GetVariable "windowsGmsaPackageUrl" }}"; + +# TLS Bootstrap Token +$global:TLSBootstrapToken = "{{GetTLSBootstrapTokenForKubeConfig}}" + +# Disable OutBoundNAT in Azure CNI configuration +$global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat" }}"); + +# Base64 representation of ZIP archive +$zippedFiles = "{{ GetKubernetesWindowsAgentFunctions }}" + +$global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" +$fipsEnabled = [System.Convert]::ToBoolean("{{ FIPSEnabled }}") + +# HNS remediator +$global:HNSRemediatorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}"); + +# Log generator +$global:LogGeneratorIntervalInMinutes = [System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}"); + +$global:EnableIncreaseDynamicPortRange = $false + +$global:RebootNeeded = $false + +# Extract cse helper script from ZIP +[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) +Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force + +# Dot-source windowscsehelper.ps1 with functions that are called in this script +. c:\AzureData\windows\windowscsehelper.ps1 +# util functions only can be used after this line, for example, Write-Log + +$global:OperationId = New-Guid + +try +{ + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + + # Exit early if the script has been executed + if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { + Write-Log "The script has been executed before, will exit without doing anything." + return + } + + # This involes using proxy, log the config before fetching packages + Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'" + # TODO update to use proxy + + $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.40.zip" + Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl" + Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage" + # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl + # but it is better to set the CSE package version in Windows CSE in AgentBaker + # since most changes in CSE package also need the change in Windows CSE in AgentBaker + # In future, AKS RP only sets the endpoint with the pacakge name, for example, https://acs-mirror.azureedge.net/aks/windows/cse/ + if ($global:CSEScriptsPackageUrl.EndsWith("/")) { + $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage + Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" + } + + # Download CSE function scripts + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" + $tempfile = 'c:\csescripts.zip' + DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE + Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" -Force + Remove-Item -Path $tempfile -Force + + # Dot-source cse scripts with functions that are called in this script + . c:\AzureData\windows\azurecnifunc.ps1 + . c:\AzureData\windows\calicofunc.ps1 + . c:\AzureData\windows\configfunc.ps1 + . c:\AzureData\windows\containerdfunc.ps1 + . c:\AzureData\windows\kubeletfunc.ps1 + . c:\AzureData\windows\kubernetesfunc.ps1 + . c:\AzureData\windows\nvidiagpudriverfunc.ps1 + + # Install OpenSSH if SSH enabled + $sshEnabled = [System.Convert]::ToBoolean("{{ WindowsSSHEnabled }}") + + if ( $sshEnabled ) { + Install-OpenSSH -SSHKeys $SSHKeys + } + + Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID + + Resize-OSDrive + + Initialize-DataDisks + + Initialize-DataDirectories + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" + Create-Directory -FullPath "c:\k" + Write-Log "Remove `+"`"+`"NT AUTHORITY\Authenticated Users`+"`"+`" write permissions on files in c:\k" + icacls.exe "c:\k" /inheritance:r + icacls.exe "c:\k" /grant:r SYSTEM:`+"`"+`(OI`+"`"+`)`+"`"+`(CI`+"`"+`)`+"`"+`(F`+"`"+`) + icacls.exe "c:\k" /grant:r BUILTIN\Administrators:`+"`"+`(OI`+"`"+`)`+"`"+`(CI`+"`"+`)`+"`"+`(F`+"`"+`) + icacls.exe "c:\k" /grant:r BUILTIN\Users:`+"`"+`(OI`+"`"+`)`+"`"+`(CI`+"`"+`)`+"`"+`(RX`+"`"+`) + Write-Log "c:\k permissions: " + icacls.exe "c:\k" + Get-ProvisioningScripts + Get-LogCollectionScripts + + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp + + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL + + $cniBinPath = $global:AzureCNIBinDir + $cniConfigPath = $global:AzureCNIConfDir + if ($global:NetworkPlugin -eq "kubenet") { + $cniBinPath = $global:CNIPath + $cniConfigPath = $global:CNIConfigPath + } + + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment + + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud + Write-AzureConfig `+"`"+` + -KubeDir $global:KubeDir `+"`"+` + -AADClientId $AADClientId `+"`"+` + -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) `+"`"+` + -TenantId $global:TenantId `+"`"+` + -SubscriptionId $global:SubscriptionId `+"`"+` + -ResourceGroup $global:ResourceGroup `+"`"+` + -Location $Location `+"`"+` + -VmType $global:VmType `+"`"+` + -SubnetName $global:SubnetName `+"`"+` + -SecurityGroupName $global:SecurityGroupName `+"`"+` + -VNetName $global:VNetName `+"`"+` + -RouteTableName $global:RouteTableName `+"`"+` + -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName `+"`"+` + -PrimaryScaleSetName $global:PrimaryScaleSetName `+"`"+` + -UseManagedIdentityExtension $global:UseManagedIdentityExtension `+"`"+` + -UserAssignedClientID $UserAssignedClientID `+"`"+` + -UseInstanceMetadata $global:UseInstanceMetadata `+"`"+` + -LoadBalancerSku $global:LoadBalancerSku `+"`"+` + -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB `+"`"+` + -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud"{{else}}$TargetEnvironment{{end}} + + # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud. + # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json + {{if IsAKSCustomCloud}} + $azureStackConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json") + $envJSON = "{{ GetBase64EncodedEnvironmentJSON }}" + [io.file]::WriteAllBytes($azureStackConfigFile, [System.Convert]::FromBase64String($envJSON)) + + Get-CACertificates + {{end}} + + Write-CACert -CACertificate $global:CACertificate `+"`"+` + -KubeDir $global:KubeDir + + if ($global:EnableCsiProxy) { + New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir + } + + if ($global:TLSBootstrapToken) { + Write-BootstrapKubeConfig -CACertificate $global:CACertificate `+"`"+` + -KubeDir $global:KubeDir `+"`"+` + -MasterFQDNPrefix $MasterFQDNPrefix `+"`"+` + -MasterIP $MasterIP `+"`"+` + -TLSBootstrapToken $global:TLSBootstrapToken + + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled + # This kubeconfig will deleted after calico installation. + # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block + Write-Log "Write temporary kube config" + } else { + Write-Log "Write kube config" + } + + Write-KubeConfig -CACertificate $global:CACertificate `+"`"+` + -KubeDir $global:KubeDir `+"`"+` + -MasterFQDNPrefix $MasterFQDNPrefix `+"`"+` + -MasterIP $MasterIP `+"`"+` + -AgentKey $AgentKey `+"`"+` + -AgentCertificate $global:AgentCertificate + + if ($global:EnableHostsConfigAgent) { + New-HostsConfigService + } + + Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin" + + # Configure network policy. + Get-HnsPsm1 -HNSModule $global:HNSModule + Import-Module $global:HNSModule + + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir `+"`"+` + -AzureCNIBinDir $global:AzureCNIBinDir `+"`"+` + -VNetCNIPluginsURL $global:VNetCNIPluginsURL + + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir `+"`"+` + -KubeDnsSearchPath $global:KubeDnsSearchPath `+"`"+` + -KubeClusterCIDR $global:KubeClusterCIDR `+"`"+` + -KubeServiceCIDR $global:KubeServiceCIDR `+"`"+` + -VNetCIDR $global:VNetCIDR `+"`"+` + -IsDualStackEnabled $global:IsDualStackEnabled `+"`"+` + -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + + if ($TargetEnvironment -ieq "AzureStackCloud") { + GenerateAzureStackCNIConfig `+"`"+` + -TenantId $global:TenantId `+"`"+` + -SubscriptionId $global:SubscriptionId `+"`"+` + -ResourceGroup $global:ResourceGroup `+"`"+` + -AADClientId $AADClientId `+"`"+` + -KubeDir $global:KubeDir `+"`"+` + -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) `+"`"+` + -NetworkAPIVersion $NetworkAPIVersion `+"`"+` + -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) `+"`"+` + -IdentitySystem "{{ GetIdentitySystem }}" + } + + New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled + + Install-KubernetesServices `+"`"+` + -KubeDir $global:KubeDir + + Set-Explorer + Adjust-PageFileSize + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" + PREPROVISION_EXTENSION + Update-ServiceFailureActions + Adjust-DynamicPortRange + Register-LogsCleanupScriptTask + Register-NodeResetScriptTask + Update-DefenderPreferences + + $windowsVersion = Get-WindowsVersion + if ($windowsVersion -ne "1809") { + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" + try { + . C:\k\windowssecuretls.ps1 + Enable-SecureTls + } + catch { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS -ErrorMessage $_ + } + } + + Enable-FIPSMode -FipsEnabled $fipsEnabled + if ($global:WindowsGmsaPackageUrl) { + Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl + } + + Check-APIServerConnectivity -MasterIP $MasterIP + + if ($global:WindowsCalicoPackageURL) { + Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp + } + + Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL + + if (Test-Path $CacheDir) + { + Write-Log "Removing aks cache directory" + Remove-Item $CacheDir -Recurse -Force + } + + if ($global:TLSBootstrapToken) { + Write-Log "Removing temporary kube config" + $kubeConfigFile = [io.path]::Combine($KubeDir, "config") + Remove-Item $kubeConfigFile + } + + Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes + + if ($global:RebootNeeded) { + Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot" + Postpone-RestartComputer + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" + Start-ScheduledTask -TaskName "k8s-restart-job" + + $timeout = 180 ## seconds + $timer = [Diagnostics.Stopwatch]::StartNew() + while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') { + # The task `+"`"+`k8s-restart-job`+"`"+` needs ~8 seconds. + if ($timer.Elapsed.TotalSeconds -gt $timeout) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds" + } + + Write-Log -Message "Waiting on NodeResetScriptTask..." + Start-Sleep -Seconds 3 + } + $timer.Stop() + Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" + } +} +catch +{ + # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block + # Ideally all exceptions will be handled and no exception will be thrown. + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_ +} +finally +{ + # Generate CSE result so it can be returned as the CSE response in csecmd.ps1 + $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date)) + Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode" + # $CSEResultFilePath is used to avoid running CSE multiple times + Set-Content -Path $CSEResultFilePath -Value $global:ExitCode -Force + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Please not use Write-Log or Logs-To-Events after Stop-Transcript + Stop-Transcript + + # Remove the parameters in the log file to avoid leaking secrets + $logs=Get-Content $LogFile | Where-Object {$_ -notmatch "^Host Application: "} + $logs | Set-Content $LogFile + + Upload-GuestVMLogs -ExitCode $global:ExitCode + if ($global:ExitCode -ne 0) { + # $JsonString = "ExitCode: |{0}|, Output: |{1}|, Error: |{2}|" + # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe. + $errorMessageLength = "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length + $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length)) + throw "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|" + } +} `) func windowsKuberneteswindowssetupPs1Bytes() ([]byte, error) { @@ -8911,466 +8910,466 @@ func windowsSendlogsPs1() (*asset, error) { return a, nil } -var _windowsWindowscsehelperPs1 = []byte(`# This script is used to define basic util functions -# It is better to define functions in the scripts under staging/cse/windows. - -# Define all exit codes in Windows CSE -# It must match ` + "`" + `[A-Z_]+` + "`" + ` -$global:WINDOWS_CSE_SUCCESS=0 -$global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY=2 -$global:WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE=3 -$global:WINDOWS_CSE_ERROR_FILE_NOT_EXIST=4 -$global:WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY=5 -$global:WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST=6 -$global:WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX=7 -$global:WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM=8 -$global:WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST=9 -$global:WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST=10 -$global:WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST=11 -$global:WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST=12 -$global:WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED=13 -$global:WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING=14 -$global:WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED=15 -$global:WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED=16 -$global:WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG=17 -$global:WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER=18 -$global:WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES=19 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES=20 -$global:WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES=21 -$global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS=22 -$global:WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE=23 -$global:WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE=24 -$global:WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION=25 -$global:WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES=26 -$global:WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS=27 -$global:WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS=28 -$global:WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP=29 -$global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER=30 -$global:WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS=31 -$global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK=32 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE=33 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE=34 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE=35 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE=36 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE=37 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE=38 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE=39 -$global:WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE=40 -$global:WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE=41 -$global:WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER=42 -$global:WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE=43 -$global:WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE=44 -$global:WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST=45 -$global:WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE=46 -$global:WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE=47 -$global:WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE=48 -$global:WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN=49 # Return this error code in csecmd.ps1 when C:\AzureData\CustomData.bin does not exist -$global:WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG=50 # Return this error code in csecmd.ps1 when C:\AzureData\CSEResult.log does not exist -$global:WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS=51 -$global:WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE=52 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED=53 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT=54 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED=55 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET=56 -$global:WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND=57 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE=58 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE=59 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION=60 -$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61 -$global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62 -$global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63 -$global:WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION=64 - -# Please add new error code for downloading new packages in RP code too -$global:ErrorCodeNames = @( - "WINDOWS_CSE_SUCCESS", - "WINDOWS_CSE_ERROR_UNKNOWN", - "WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY", - "WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE", - "WINDOWS_CSE_ERROR_FILE_NOT_EXIST", - "WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY", - "WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST", - "WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX", - "WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM", - "WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST", - "WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST", - "WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST", - "WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST", - "WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED", - "WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING", - "WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED", - "WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED", - "WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG", - "WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER", - "WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES", - "WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES", - "WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES", - "WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS", - "WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE", - "WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE", - "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION", - "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES", - "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS", - "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS", - "WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP", - "WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER", - "WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS", - "WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK", - "WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE", - "WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE", - "WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE", - "WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE", - "WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER", - "WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE", - "WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE", - "WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST", - "WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE", - "WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE", - "WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE", - "WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN", - "WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG", - "WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS", - "WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET", - "WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION", - "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE", - "WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG", - "WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP", - "WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION" -) - -# NOTE: KubernetesVersion does not contain "v" -$global:MinimalKubernetesVersionWithLatestContainerd = "1.28.0" # Will change it to the correct version when we support new Windows containerd version -# DEPRECATED: The contianerd package url will be set in AKS RP code. We will remove the following variables in the future. -$global:StableContainerdPackage = "v1.6.21-azure.1/binaries/containerd-v1.6.21-azure.1-windows-amd64.tar.gz" -# The latest containerd version -$global:LatestContainerdPackage = "v1.7.9-azure.1/binaries/containerd-v1.7.9-azure.1-windows-amd64.tar.gz" - -$global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" -$global:TaskName = "" -$global:TaskTimeStamp = "" - -# This filter removes null characters (\0) which are captured in nssm.exe output when logged through powershell -filter RemoveNulls { $_ -replace '\0', '' } - -filter Timestamp { "$(Get-Date -Format o): $_" } - -function Write-Log($message) { - $msg = $message | Timestamp - Write-Output $msg -} - -function DownloadFileOverHttp { - Param( - [Parameter(Mandatory = $true)][string] - $Url, - [Parameter(Mandatory = $true)][string] - $DestinationPath, - [Parameter(Mandatory = $true)][int] - $ExitCode - ) - - # First check to see if a file with the same name is already cached on the VHD - $fileName = [IO.Path]::GetFileName($Url) - - $search = @() - if (Test-Path $global:CacheDir) { - $search = [IO.Directory]::GetFiles($global:CacheDir, $fileName, [IO.SearchOption]::AllDirectories) - } - - if ($search.Count -ne 0) { - Write-Log "Using cached version of $fileName - Copying file from $($search[0]) to $DestinationPath" - Copy-Item -Path $search[0] -Destination $DestinationPath -Force - } - else { - $secureProtocols = @() - $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) - - foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) { - if ($insecureProtocols -notcontains $protocol) { - $secureProtocols += $protocol - } - } - [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols - - $oldProgressPreference = $ProgressPreference - $ProgressPreference = 'SilentlyContinue' - - $downloadTimer = [System.Diagnostics.Stopwatch]::StartNew() - try { - $args = @{Uri=$Url; Method="Get"; OutFile=$DestinationPath} - Retry-Command -Command "Invoke-RestMethod" -Args $args -Retries 5 -RetryDelaySeconds 10 - } catch { - Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Failed in downloading $Url. Error: $_" - } - $downloadTimer.Stop() - - if ($global:AppInsightsClient -ne $null) { - $event = New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" - $event.Name = "FileDownload" - $event.Properties["FileName"] = $fileName - $event.Metrics["DurationMs"] = $downloadTimer.ElapsedMilliseconds - $global:AppInsightsClient.TrackEvent($event) - } - - $ProgressPreference = $oldProgressPreference - Write-Log "Downloaded file $Url to $DestinationPath" - } -} - -function Set-ExitCode -{ - Param( - [Parameter(Mandatory=$true)][int] - $ExitCode, - [Parameter(Mandatory=$true)][string] - $ErrorMessage - ) - Write-Log "Set ExitCode to $ExitCode and exit. Error: $ErrorMessage" - $global:ExitCode=$ExitCode - # we use | as the separator as a workaround since " or ' do not work as expected per the testings - $global:ErrorMessage=($ErrorMessage -replace '\|', '%7C') - exit $ExitCode -} - -function Postpone-RestartComputer -{ - Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" - $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command ` + "`" + `"Restart-Computer -Force` + "`" + `"" - $principal = New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest - # trigger this task once - $trigger = New-JobTrigger -At (Get-Date).AddSeconds(15).DateTime -Once - $definition = New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "Restart computer after provisioning the VM" - Register-ScheduledTask -TaskName "restart-computer" -InputObject $definition - Write-Log "Created an one-time task to restart the VM" -} - -function Create-Directory -{ - Param( - [Parameter(Mandatory=$true)][string] - $FullPath, - [Parameter(Mandatory=$false)][string] - $DirectoryUsage = "general purpose" - ) - - if (-Not (Test-Path $FullPath)) { - Write-Log "Create directory $FullPath for $DirectoryUsage" - New-Item -ItemType Directory -Path $FullPath > $null - } else { - Write-Log "Directory $FullPath for $DirectoryUsage exists" - } -} - -# https://stackoverflow.com/a/34559554/697126 -function New-TemporaryDirectory { - $parent = [System.IO.Path]::GetTempPath() - [string] $name = [System.Guid]::NewGuid() - New-Item -ItemType Directory -Path (Join-Path $parent $name) -} - -function Retry-Command { - Param( - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] - $Command, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][hashtable] - $Args, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] - $Retries, - [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] - $RetryDelaySeconds - ) - - for ($i = 0; ; ) { - try { - # Do not log Args since Args may contain sensitive data - Write-Log "Retry $i : $command" - return & $Command @Args - } - catch { - $i++ - if ($i -ge $Retries) { - throw $_ - } - Start-Sleep $RetryDelaySeconds - } - } -} - -function Invoke-Executable { - Param( - [Parameter(Mandatory=$true)][string] - $Executable, - [Parameter(Mandatory=$true)][string[]] - $ArgList, - [Parameter(Mandatory=$true)][int] - $ExitCode, - [int[]] - $AllowedExitCodes = @(0), - [int] - $Retries = 0, - [int] - $RetryDelaySeconds = 1 - ) - - for ($i = 0; $i -le $Retries; $i++) { - Write-Log "$i - Running $Executable $ArgList ..." - & $Executable $ArgList - if ($LASTEXITCODE -notin $AllowedExitCodes) { - Write-Log "$Executable returned unsuccessfully with exit code $LASTEXITCODE" - Start-Sleep -Seconds $RetryDelaySeconds - continue - } - else { - Write-Log "$Executable returned successfully" - return - } - } - - Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Exhausted retries for $Executable $ArgList" -} - -function Assert-FileExists { - Param( - [Parameter(Mandatory = $true)][string] - $Filename, - [Parameter(Mandatory = $true)][int] - $ExitCode - ) - - if (-Not (Test-Path $Filename)) { - Set-ExitCode -ExitCode $ExitCode -ErrorMessage "$Filename does not exist" - } -} - -function Get-WindowsBuildNumber { - return (Get-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion").CurrentBuild -} - -function Get-WindowsVersion { - $buildNumber = Get-WindowsBuildNumber - switch ($buildNumber) { - "17763" { return "1809" } - "20348" { return "ltsc2022" } - "25398" { return "23H2" } - {$_ -ge "25399" -and $_ -le "30397"} { return "test2025" } - Default { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" - } - } -} - -function Get-WindowsPauseVersion { - $buildNumber = Get-WindowsBuildNumber - switch ($buildNumber) { - "17763" { return "1809" } - "20348" { return "ltsc2022" } - "25398" { return "ltsc2022" } - {$_ -ge "25399" -and $_ -le "30397"} { return "ltsc2022" } - Default { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" - } - } -} - -function Install-Containerd-Based-On-Kubernetes-Version { - Param( - [Parameter(Mandatory = $true)][string] - $ContainerdUrl, - [Parameter(Mandatory = $true)][string] - $CNIBinDir, - [Parameter(Mandatory = $true)][string] - $CNIConfDir, - [Parameter(Mandatory = $true)][string] - $KubeDir, - [Parameter(Mandatory = $true)][string] - $KubernetesVersion - ) - - Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion" - - # In the past, $global:ContainerdUrl is a full URL to download Windows containerd package. - # Example: "https://acs-mirror.azureedge.net/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" - # To support multiple containerd versions, we only set the endpoint in $global:ContainerdUrl. - # Example: "https://acs-mirror.azureedge.net/containerd/windows/" - # We only set containerd package based on kubernetes version when $global:ContainerdUrl ends with "/" so we support: - # 1. Current behavior to set the full URL - # 2. Setting containerd package in toggle for test purpose or hotfix - if ($ContainerdUrl.EndsWith("/")) { - Write-Log "ContainerdURL is $ContainerdUrl" - $containerdPackage=$global:StableContainerdPackage - if (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) { - $containerdPackage=$global:LatestContainerdPackage - Write-Log "Kubernetes version $KubernetesVersion is greater than or equal to $global:MinimalKubernetesVersionWithLatestContainerd so the latest containerd version $containerdPackage is used" - } else { - Write-Log "Kubernetes version $KubernetesVersion is less than $global:MinimalKubernetesVersionWithLatestContainerd so the stable containerd version $containerdPackage is used" - } - $ContainerdUrl = $ContainerdUrl + $containerdPackage - } - Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" - Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir -} - -function Logs-To-Event { - Param( - [Parameter(Mandatory = $true)][string] - $TaskName, - [Parameter(Mandatory = $true)][string] - $TaskMessage - ) - $eventLevel="Informational" - if ($global:ExitCode -ne 0) { - $eventLevel="Error" - } - - $eventsFileName=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - $currentTime=$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff") - - $lastTaskName = "" - $lastTaskDuration = 0 - if ($global:TaskTimeStamp -ne "") { - $lastTaskName = $global:TaskName - $lastTaskDuration = $(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) - } - - $global:TaskName = $TaskName - $global:TaskTimeStamp = $currentTime - - Write-Log "$global:TaskName - $TaskMessage" - $TaskMessage = (echo $TaskMessage | ConvertTo-Json) - $messageJson = @" - { - "HostName": "$env:computername", - "LastTaskName": "$lastTaskName", - "LastTaskDuration": "$lastTaskDuration", - "CurrentTaskMessage": $TaskMessage - } -"@ - $messageJson = (echo $messageJson | ConvertTo-Json) - - $jsonString = @" - { - "Timestamp": "$global:TaskTimeStamp", - "OperationId": "$global:OperationId", - "Version": "1.10", - "TaskName": "$global:TaskName", - "EventLevel": "$eventLevel", - "Message": $messageJson - } -"@ - echo $jsonString | Set-Content ${global:EventsLoggingDir}${eventsFileName}.json +var _windowsWindowscsehelperPs1 = []byte(`# This script is used to define basic util functions +# It is better to define functions in the scripts under staging/cse/windows. + +# Define all exit codes in Windows CSE +# It must match `+"`"+`[A-Z_]+`+"`"+` +$global:WINDOWS_CSE_SUCCESS=0 +$global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY=2 +$global:WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE=3 +$global:WINDOWS_CSE_ERROR_FILE_NOT_EXIST=4 +$global:WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY=5 +$global:WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST=6 +$global:WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX=7 +$global:WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM=8 +$global:WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST=9 +$global:WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST=10 +$global:WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST=11 +$global:WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST=12 +$global:WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED=13 +$global:WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING=14 +$global:WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED=15 +$global:WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED=16 +$global:WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG=17 +$global:WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER=18 +$global:WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES=19 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES=20 +$global:WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES=21 +$global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS=22 +$global:WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE=23 +$global:WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE=24 +$global:WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION=25 +$global:WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES=26 +$global:WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS=27 +$global:WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS=28 +$global:WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP=29 +$global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER=30 +$global:WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS=31 +$global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK=32 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE=33 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE=34 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE=35 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE=36 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE=37 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE=38 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE=39 +$global:WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE=40 +$global:WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE=41 +$global:WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER=42 +$global:WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE=43 +$global:WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE=44 +$global:WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST=45 +$global:WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE=46 +$global:WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE=47 +$global:WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE=48 +$global:WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN=49 # Return this error code in csecmd.ps1 when C:\AzureData\CustomData.bin does not exist +$global:WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG=50 # Return this error code in csecmd.ps1 when C:\AzureData\CSEResult.log does not exist +$global:WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS=51 +$global:WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE=52 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED=53 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT=54 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED=55 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET=56 +$global:WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND=57 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE=58 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE=59 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION=60 +$global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61 +$global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62 +$global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63 +$global:WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION=64 + +# Please add new error code for downloading new packages in RP code too +$global:ErrorCodeNames = @( + "WINDOWS_CSE_SUCCESS", + "WINDOWS_CSE_ERROR_UNKNOWN", + "WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY", + "WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE", + "WINDOWS_CSE_ERROR_FILE_NOT_EXIST", + "WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY", + "WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST", + "WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX", + "WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM", + "WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST", + "WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST", + "WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST", + "WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST", + "WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED", + "WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING", + "WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED", + "WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED", + "WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG", + "WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER", + "WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES", + "WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES", + "WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES", + "WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS", + "WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE", + "WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE", + "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION", + "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES", + "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS", + "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS", + "WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP", + "WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER", + "WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS", + "WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK", + "WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE", + "WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE", + "WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE", + "WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE", + "WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER", + "WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE", + "WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE", + "WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST", + "WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE", + "WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE", + "WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE", + "WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN", + "WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG", + "WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS", + "WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET", + "WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION", + "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE", + "WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG", + "WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP", + "WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION" +) + +# NOTE: KubernetesVersion does not contain "v" +$global:MinimalKubernetesVersionWithLatestContainerd = "1.28.0" # Will change it to the correct version when we support new Windows containerd version +# DEPRECATED: The contianerd package url will be set in AKS RP code. We will remove the following variables in the future. +$global:StableContainerdPackage = "v1.6.21-azure.1/binaries/containerd-v1.6.21-azure.1-windows-amd64.tar.gz" +# The latest containerd version +$global:LatestContainerdPackage = "v1.7.9-azure.1/binaries/containerd-v1.7.9-azure.1-windows-amd64.tar.gz" + +$global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" +$global:TaskName = "" +$global:TaskTimeStamp = "" + +# This filter removes null characters (\0) which are captured in nssm.exe output when logged through powershell +filter RemoveNulls { $_ -replace '\0', '' } + +filter Timestamp { "$(Get-Date -Format o): $_" } + +function Write-Log($message) { + $msg = $message | Timestamp + Write-Output $msg +} + +function DownloadFileOverHttp { + Param( + [Parameter(Mandatory = $true)][string] + $Url, + [Parameter(Mandatory = $true)][string] + $DestinationPath, + [Parameter(Mandatory = $true)][int] + $ExitCode + ) + + # First check to see if a file with the same name is already cached on the VHD + $fileName = [IO.Path]::GetFileName($Url) + + $search = @() + if (Test-Path $global:CacheDir) { + $search = [IO.Directory]::GetFiles($global:CacheDir, $fileName, [IO.SearchOption]::AllDirectories) + } + + if ($search.Count -ne 0) { + Write-Log "Using cached version of $fileName - Copying file from $($search[0]) to $DestinationPath" + Copy-Item -Path $search[0] -Destination $DestinationPath -Force + } + else { + $secureProtocols = @() + $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) + + foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) { + if ($insecureProtocols -notcontains $protocol) { + $secureProtocols += $protocol + } + } + [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols + + $oldProgressPreference = $ProgressPreference + $ProgressPreference = 'SilentlyContinue' + + $downloadTimer = [System.Diagnostics.Stopwatch]::StartNew() + try { + $args = @{Uri=$Url; Method="Get"; OutFile=$DestinationPath} + Retry-Command -Command "Invoke-RestMethod" -Args $args -Retries 5 -RetryDelaySeconds 10 + } catch { + Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Failed in downloading $Url. Error: $_" + } + $downloadTimer.Stop() + + if ($global:AppInsightsClient -ne $null) { + $event = New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" + $event.Name = "FileDownload" + $event.Properties["FileName"] = $fileName + $event.Metrics["DurationMs"] = $downloadTimer.ElapsedMilliseconds + $global:AppInsightsClient.TrackEvent($event) + } + + $ProgressPreference = $oldProgressPreference + Write-Log "Downloaded file $Url to $DestinationPath" + } +} + +function Set-ExitCode +{ + Param( + [Parameter(Mandatory=$true)][int] + $ExitCode, + [Parameter(Mandatory=$true)][string] + $ErrorMessage + ) + Write-Log "Set ExitCode to $ExitCode and exit. Error: $ErrorMessage" + $global:ExitCode=$ExitCode + # we use | as the separator as a workaround since " or ' do not work as expected per the testings + $global:ErrorMessage=($ErrorMessage -replace '\|', '%7C') + exit $ExitCode +} + +function Postpone-RestartComputer +{ + Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" + $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command `+"`"+`"Restart-Computer -Force`+"`"+`"" + $principal = New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest + # trigger this task once + $trigger = New-JobTrigger -At (Get-Date).AddSeconds(15).DateTime -Once + $definition = New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "Restart computer after provisioning the VM" + Register-ScheduledTask -TaskName "restart-computer" -InputObject $definition + Write-Log "Created an one-time task to restart the VM" +} + +function Create-Directory +{ + Param( + [Parameter(Mandatory=$true)][string] + $FullPath, + [Parameter(Mandatory=$false)][string] + $DirectoryUsage = "general purpose" + ) + + if (-Not (Test-Path $FullPath)) { + Write-Log "Create directory $FullPath for $DirectoryUsage" + New-Item -ItemType Directory -Path $FullPath > $null + } else { + Write-Log "Directory $FullPath for $DirectoryUsage exists" + } +} + +# https://stackoverflow.com/a/34559554/697126 +function New-TemporaryDirectory { + $parent = [System.IO.Path]::GetTempPath() + [string] $name = [System.Guid]::NewGuid() + New-Item -ItemType Directory -Path (Join-Path $parent $name) +} + +function Retry-Command { + Param( + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] + $Command, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][hashtable] + $Args, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] + $Retries, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][int] + $RetryDelaySeconds + ) + + for ($i = 0; ; ) { + try { + # Do not log Args since Args may contain sensitive data + Write-Log "Retry $i : $command" + return & $Command @Args + } + catch { + $i++ + if ($i -ge $Retries) { + throw $_ + } + Start-Sleep $RetryDelaySeconds + } + } +} + +function Invoke-Executable { + Param( + [Parameter(Mandatory=$true)][string] + $Executable, + [Parameter(Mandatory=$true)][string[]] + $ArgList, + [Parameter(Mandatory=$true)][int] + $ExitCode, + [int[]] + $AllowedExitCodes = @(0), + [int] + $Retries = 0, + [int] + $RetryDelaySeconds = 1 + ) + + for ($i = 0; $i -le $Retries; $i++) { + Write-Log "$i - Running $Executable $ArgList ..." + & $Executable $ArgList + if ($LASTEXITCODE -notin $AllowedExitCodes) { + Write-Log "$Executable returned unsuccessfully with exit code $LASTEXITCODE" + Start-Sleep -Seconds $RetryDelaySeconds + continue + } + else { + Write-Log "$Executable returned successfully" + return + } + } + + Set-ExitCode -ExitCode $ExitCode -ErrorMessage "Exhausted retries for $Executable $ArgList" +} + +function Assert-FileExists { + Param( + [Parameter(Mandatory = $true)][string] + $Filename, + [Parameter(Mandatory = $true)][int] + $ExitCode + ) + + if (-Not (Test-Path $Filename)) { + Set-ExitCode -ExitCode $ExitCode -ErrorMessage "$Filename does not exist" + } +} + +function Get-WindowsBuildNumber { + return (Get-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion").CurrentBuild +} + +function Get-WindowsVersion { + $buildNumber = Get-WindowsBuildNumber + switch ($buildNumber) { + "17763" { return "1809" } + "20348" { return "ltsc2022" } + "25398" { return "23H2" } + {$_ -ge "25399" -and $_ -le "30397"} { return "test2025" } + Default { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" + } + } +} + +function Get-WindowsPauseVersion { + $buildNumber = Get-WindowsBuildNumber + switch ($buildNumber) { + "17763" { return "1809" } + "20348" { return "ltsc2022" } + "25398" { return "ltsc2022" } + {$_ -ge "25399" -and $_ -le "30397"} { return "ltsc2022" } + Default { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER -ErrorMessage "Failed to find the windows build number: $buildNumber" + } + } +} + +function Install-Containerd-Based-On-Kubernetes-Version { + Param( + [Parameter(Mandatory = $true)][string] + $ContainerdUrl, + [Parameter(Mandatory = $true)][string] + $CNIBinDir, + [Parameter(Mandatory = $true)][string] + $CNIConfDir, + [Parameter(Mandatory = $true)][string] + $KubeDir, + [Parameter(Mandatory = $true)][string] + $KubernetesVersion + ) + + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion" + + # In the past, $global:ContainerdUrl is a full URL to download Windows containerd package. + # Example: "https://acs-mirror.azureedge.net/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" + # To support multiple containerd versions, we only set the endpoint in $global:ContainerdUrl. + # Example: "https://acs-mirror.azureedge.net/containerd/windows/" + # We only set containerd package based on kubernetes version when $global:ContainerdUrl ends with "/" so we support: + # 1. Current behavior to set the full URL + # 2. Setting containerd package in toggle for test purpose or hotfix + if ($ContainerdUrl.EndsWith("/")) { + Write-Log "ContainerdURL is $ContainerdUrl" + $containerdPackage=$global:StableContainerdPackage + if (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) { + $containerdPackage=$global:LatestContainerdPackage + Write-Log "Kubernetes version $KubernetesVersion is greater than or equal to $global:MinimalKubernetesVersionWithLatestContainerd so the latest containerd version $containerdPackage is used" + } else { + Write-Log "Kubernetes version $KubernetesVersion is less than $global:MinimalKubernetesVersionWithLatestContainerd so the stable containerd version $containerdPackage is used" + } + $ContainerdUrl = $ContainerdUrl + $containerdPackage + } + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" + Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir +} + +function Logs-To-Event { + Param( + [Parameter(Mandatory = $true)][string] + $TaskName, + [Parameter(Mandatory = $true)][string] + $TaskMessage + ) + $eventLevel="Informational" + if ($global:ExitCode -ne 0) { + $eventLevel="Error" + } + + $eventsFileName=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $currentTime=$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff") + + $lastTaskName = "" + $lastTaskDuration = 0 + if ($global:TaskTimeStamp -ne "") { + $lastTaskName = $global:TaskName + $lastTaskDuration = $(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) + } + + $global:TaskName = $TaskName + $global:TaskTimeStamp = $currentTime + + Write-Log "$global:TaskName - $TaskMessage" + $TaskMessage = (echo $TaskMessage | ConvertTo-Json) + $messageJson = @" + { + "HostName": "$env:computername", + "LastTaskName": "$lastTaskName", + "LastTaskDuration": "$lastTaskDuration", + "CurrentTaskMessage": $TaskMessage + } +"@ + $messageJson = (echo $messageJson | ConvertTo-Json) + + $jsonString = @" + { + "Timestamp": "$global:TaskTimeStamp", + "OperationId": "$global:OperationId", + "Version": "1.10", + "TaskName": "$global:TaskName", + "EventLevel": "$eventLevel", + "Message": $messageJson + } +"@ + echo $jsonString | Set-Content ${global:EventsLoggingDir}${eventsFileName}.json }`) func windowsWindowscsehelperPs1Bytes() ([]byte, error) { @@ -9550,13 +9549,11 @@ var _bindata = map[string]func() (*asset, error){ // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the // following hierarchy: -// -// data/ -// foo.txt -// img/ -// a.png -// b.png -// +// data/ +// foo.txt +// img/ +// a.png +// b.png // then AssetDir("data") would return []string{"foo.txt", "img"} // AssetDir("data/img") would return []string{"a.png", "b.png"} // AssetDir("foo.txt") and AssetDir("notexist") would return an error