From 167aae67c102530768a7108443e24a7767b9ad9b Mon Sep 17 00:00:00 2001 From: Junjie Zhang Date: Thu, 21 Dec 2023 07:45:22 +0000 Subject: [PATCH] chore: support cse duration log for windows --- parts/windows/kuberneteswindowssetup.ps1 | 76 ++++------ parts/windows/windowscsehelper.ps1 | 58 +++++++- .../AKSWindows2019+CustomCloud/CustomData | 78 +++++----- .../AKSWindows2019+CustomVnet/CustomData | 78 +++++----- .../CustomData | 78 +++++----- .../testdata/AKSWindows2019+K8S116/CustomData | 78 +++++----- .../testdata/AKSWindows2019+K8S117/CustomData | 78 +++++----- .../testdata/AKSWindows2019+K8S118/CustomData | 78 +++++----- .../AKSWindows2019+K8S119+CSI/CustomData | 78 +++++----- .../AKSWindows2019+K8S119+FIPS/CustomData | 78 +++++----- .../testdata/AKSWindows2019+K8S119/CustomData | 78 +++++----- .../CustomData | 78 +++++----- .../AKSWindows2019+ManagedIdentity/CustomData | 78 +++++----- .../AKSWindows2019+SecurityProfile/CustomData | 78 +++++----- pkg/templates/templates_generated.go | 136 ++++++++++++------ staging/cse/windows/azurecnifunc.ps1 | 7 + staging/cse/windows/calicofunc.ps1 | 1 + staging/cse/windows/configfunc.ps1 | 21 +++ staging/cse/windows/kubeletfunc.ps1 | 9 ++ staging/cse/windows/kubernetesfunc.ps1 | 8 ++ staging/cse/windows/nvidiagpudriverfunc.ps1 | 1 + 21 files changed, 609 insertions(+), 644 deletions(-) diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1 index 08195a54360..f0484d7d459 100644 --- a/parts/windows/kuberneteswindowssetup.ps1 +++ b/parts/windows/kuberneteswindowssetup.ps1 @@ -216,9 +216,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -241,13 +243,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -261,22 +264,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("{{ WindowsSSHEnabled }}") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -287,25 +286,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -338,22 +335,20 @@ try Get-CACertificates {{end}} - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -368,9 +363,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -379,12 +373,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -392,7 +385,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -407,32 +400,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" PREPROVISION_EXTENSION - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -444,19 +430,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -470,8 +454,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -509,6 +493,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1 index 290e82d204f..8a2480ff6a5 100644 --- a/parts/windows/windowscsehelper.ps1 +++ b/parts/windows/windowscsehelper.ps1 @@ -71,6 +71,9 @@ $global:StableContainerdPackage = "v1.6.21-azure.1/binaries/containerd-v1.6.21-a # The latest containerd version $global:LatestContainerdPackage = "v1.7.9-azure.1/binaries/containerd-v1.7.9-azure.1-windows-amd64.tar.gz" +$global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" +$global:TaskName = "" +$global:TaskTimeStamp = "" # This filter removes null characters (\0) which are captured in nssm.exe output when logged through powershell filter RemoveNulls { $_ -replace '\0', '' } @@ -156,7 +159,7 @@ function Set-ExitCode function Postpone-RestartComputer { - Write-Log "Creating an one-time task to restart the VM" + Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command `"Restart-Computer -Force`"" $principal = New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest # trigger this task once @@ -308,6 +311,8 @@ function Install-Containerd-Based-On-Kubernetes-Version { $KubernetesVersion ) + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion" + # In the past, $global:ContainerdUrl is a full URL to download Windows containerd package. # Example: "https://acs-mirror.azureedge.net/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" # To support multiple containerd versions, we only set the endpoint in $global:ContainerdUrl. @@ -326,5 +331,56 @@ function Install-Containerd-Based-On-Kubernetes-Version { } $ContainerdUrl = $ContainerdUrl + $containerdPackage } + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir +} + +function Logs-To-Event { + Param( + [Parameter(Mandatory = $true)][string] + $TaskName, + [Parameter(Mandatory = $true)][string] + $TaskMessage + ) + $eventLevel="Informational" + if ($global:ExitCode -ne 0) { + $eventLevel="Error" + } + + $eventsFileName=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $currentTime=$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff") + + $lastTaskName = "" + $lastTaskDuration = 0 + if ($global:TaskTimeStamp -ne "") { + $lastTaskName = $global:TaskName + $lastTaskDuration = $(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) + } + + $global:TaskName = $TaskName + $global:TaskTimeStamp = $currentTime + + Write-Log "$global:TaskName - $TaskMessage" + $TaskMessage = (echo $TaskMessage | ConvertTo-Json) + $messageJson = @" + { + "HostName": "$env:computername", + "LastTaskName": "$lastTaskName", + "LastTaskDuration": "$lastTaskDuration", + "CurrentTaskMessage": $TaskMessage + } +"@ + $messageJson = (echo $messageJson | ConvertTo-Json) + + $jsonString = @" + { + "Timestamp": "$global:TaskTimeStamp", + "OperationId": "$global:OperationId", + "Version": "1.10", + "TaskName": "$global:TaskName", + "EventLevel": "$eventLevel", + "Message": $messageJson + } +"@ + echo $jsonString | Set-Content ${global:EventsLoggingDir}${eventsFileName}.json } \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData index 92ca5fd4dc1..8a76f5f4abf 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -334,22 +331,20 @@ try Get-CACertificates - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -364,9 +359,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -375,12 +369,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -388,7 +381,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -403,32 +396,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -440,19 +426,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -466,8 +450,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -505,6 +489,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData index 660d1dacc20..d88021491a5 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData index b3b6bf9c8a5..41d5646a6dd 100644 --- a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData index 5e4d8a7e2c1..fce29df9f1f 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData index 69af49cc8fa..dc18f0e0801 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData index fec356c2a7d..fde37032bb5 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData index e7862c17d82..8aa9475ae81 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData index 84305166f42..1488ed79a3e 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("true") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData index 975e1d89169..0574c0b1e5f 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData index b0aeeb035e6..033024f7936 100644 --- a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "07401b.f395accd246ae52d" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData index e1878a8fbaa..9035acc9fa3 100644 --- a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData index d16c3669a4e..bf1826375cc 100644 --- a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData @@ -189,7 +189,7 @@ $global:TLSBootstrapToken = "" $global:IsDisableWindowsOutboundNat = [System.Convert]::ToBoolean("false"); # Base64 representation of ZIP archive -$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb3XLbOpK+91N0yaqJXSeU/+LkxKe4NQwF2xxLpIqk7GSSFAemYAkbiuACoB1PNu++BfBHpCzRPpnZi8mFooj9dTcaH7obALML4YIKEDGnmQQqIBdkBpLBjNzRlMAtFjSGXNIE7vI0lpSlYmcXHC17S6QkvCFdiwBNQS5IqVdAns4IByHxnKbzg1iQgweaztiDGOzs7MKwQOMkAfKdSojZjGgVN4UQ2AHa6c8TdouTsxvHHXo3QWQHKEK+7/nR1L1yvRvXPIJdOGcc8pR8z0gsyQwI54xDjPP5QsLto/YpxjJewG3C4m/Kxrf8lvCUSCJKlwSReTbIxFGHyaF34448axidOyMU3TjhZeSj0P9kHneAHPfau0IR+ojsaWh9GCHzpENaa3a9MEIfnSA033SI2pfIvoqsiRMFyL9GfmR7rovs0Ll2wk/maQdyYk0DFDlj66Jp620H4gKFUTD94KIwmvjo3PlovuuUdpFvhSgKvSvkRueeH1n+2Py9A+Ki8MbzryLHDZF/btkoaHj2/gVAa2hNQuQ3UEeHHbCx5VoXaIzcMHImTVDX/NvWyLE9HW7HRpFl297UDZvoLiLYnhtajov8oUY4bhBaoxEamkddhFhD+VPXddwL86iLGd4EuUFwuW6mixIV5Nzx0Y01Gmms7bnnzsXUV+AudjjutTVyhtHE8q0xUrPguJH196mPShXmURddXC8aevYV8qPQiz5MndGwZGg9dvOoizuKnLYV2cgPnXPHtkIUmEddnKnX8TrquIsyaDwJPz2FdBEGuWrBRwGyVSzCUWAedzHkYhxYEfo4sdxhZPn2pXONzOMuchSAwsjEu0F+cIlGo2jiO9fOCF0g87iLJhodoDDy0YUThP6naIL8sRMEjueax11seYq8tkZTFY3OJKJQznji+WFk2xfoGrlhYB53ZpI2xLq6nkxG0wvHrcCdOcULo3Nv6g7bq9087swnNahgojsdf0C+edLFjBVm4nvXjoqf415Ege07kzAwT7ooEoSWr1LIEEU+UiEtUFFoBVfmSRdbViwOUDSx7CvrApknXXSpEVfTD8h3UYiCFbCLKStTrrNCdDGkRly6QTT2hlNV97rI0ViTOsnWVrroUYM0T2pIFykaUXPUdH38tMK9LGesEnIFfNNFDjWnoT2Jhp9ca+zYkeayb7kK10WMgn5lYlzPh2+6iDGZqhywqvHmmy5SFHZC66IN6ew8ViH44LiW/6nqVroYUcUBfbRH0yFqxaGLGAo3HW6OXxc3Ktwme91ZI7KnQeiNo6EVWmqA5pv3sAs+kTlX/S0VVYPJZkT1krEg8XKmekd4WJAU7LMv1j9zToZY4i92LiRbqq+DW5rCjBEBKZOq5RXyGTcCnROmozAaeRfm6eG/4EWAfCLyRA4SNn+5E7Y3+aRsR7Y3GqnW0nPrrHbaRV4fBc7fUeQF0dBXVey0s+xNpoWYX3UrlrZ0bjmqbzntrIBbsKEzRt40NE87C+AW8PU40u6rvB5MJ4o5yo/OirhF1dQvWqkAheZpZ3GcTKPgaho57rm3KijmaWd13GJ0tU+xnNHUR+ZpZ/vU1FI0coFz4VqhRnblxG320UcbTdQ3821XZnwuZugjMt8e7ai9ouuF6Ayu6k3bNeGCssZyilkqMU2hd9+rLY5pSpc4eYK6oXIxwpIIaRcowmdgQu9ocPz74LAHu3BDkwTiBU7nBKhUW129hWSck1jCfWlcL7MHAiLPMsYlpOSh3rvGK82leO1WIPFtQlamJzj+hudEeXB/NHg7OD4ysFq4g6ODW5piTok4WKkz1mSMcv9q4OXs7ZuBxHww/2dvR+3wCSR6lF3OrMeh7cy7wfvnfGmIbHFlZ6c8brijiSQcOFmyezVteRFkjmNJuIC9L4f78LCg8QIwVxv2TOaczFRuS4VYDsh3AiyXWS6LyCdsPiczkAvO8vkCMvZAuFiQJNkpDfnakJsniYAf0I/A4CRLcEzg1ZfDV6/h1Sv4uVMJh3RJhMTLDH5Ar793QaQxxJKAcc74Ektg+2fQj3oaUZ53wA2nkhgjNt/rL4kQeE724ccOAEB/KeZgQvUz/O9Kv35eIL1iNEp4p6l3yB7ShOHZOU2Id0/4pZRZqXiCOV7u6a/qz2f9byIJ3xvjdIYl44/KruQ52f/6WUhO0/nXWrw/5cnrXwYPiZA0xcrFCZaLFyuiqWxoQd+ptNmM6F/2d/Rfu3BOuSLqgsTf1HIThAC9A6w4Q+CBykVxsISXBFL1QQXghBM8e4QYxwsyA1acPV1fDosZUEhXiZrw2fEGyuOvZ2cXRJ6XD/ZUNEoH+oJgHi/AhL/u7etf6B3shURIQwGhWi62sjWkvJrnNlbZGVKVIxh/XBkTe+vw1yv3XmtUoFV4mYrt17MzK0kqPZSIwqGfO7VfpcWBzfJUgpESOGw6VNMSelNB03kVoSpvsbtGdAywWfaopHSo7zhbQr+y8Pnw676ajvWp79W2FNhwJFlCGagaCEYD9ESDXlcxKUemPkkiSDuqcc7JhDPJYpaIxtToxzTdJPA5eBSSLAcukYNAPafysZIIHzPy9eyskBiSO5wn8jW8BCGSk5In6s8d4wTHC9jrZ6WcSlGVHpTmy2Lqr3GSE/G8S/vNuYNqip+Oz0iZLHOvgNr2Onhj7H4zV4CW9M+dp9/aDvN7GpMJo6kc4xTPCVcBWRuGWutrJlfh6rNkNuFszokQE07uCCdprFZl/+mvK9BGxKuAJiSVyaOqVzTNyauGmVmZNFWi5WoxlsMYUjxPmZA0FoNAsuwBy1hlgkBiLl3y0CCV5I9r0exjPtfM+jHl1FQJ4w8YE7lgM7N3QWTvD/ByvcbNdYL/bOnxieSPhs2WS5zOoP7Sc9J79o0YPhGyUNsDw1ImC8OGwlEi4LT4+jgkCX4MSMzSmYCjw9Xslefbbe8DIo0q3cLqW7/xm9pCjMsi1TvHNCkKbhVMlRfUqAegJXUJ3ECZdvB1mPcaK0YTukyBVpY5qaDzhRR2QkmZv/qqGVincp/cq+cmuOTB8G7/W/VevTGNORPsTg6sLEtorANeaRyozY4ih+opxAApfEgSslSx621QPigrRE/NYVV2NwpOOMsIl5SIz72qgvS+KhpXqXQTaqzmLxafe8Oca0/HogC1A4YSnAkyG9MkoaKY3ba2bcEbhBzH3/Q49wqT+43peWY5bV6Zm8pIFRkyK6qEosT2uvCz1cw0Sbjz4ibG7G4eunsPc2sL0+S7/rUIV2OoAZFQrw81xPofasGS71Su1kJTWzH0aqIqkNlud+rHDaD51Cl9HbZCNqM5YUJmLC1SBubSZsssV/1rGdrGSGxOsFQLGKegEJIuCUgsdIPFC3jRMo1L53Fho1hwgeoZ8oTMQiy+WcUTA30ncS4J9FbdturLi6yVL9Vy7a3S2z96pZdG7WZR9v/RKy1mnKYxzXCyyeikfmhMBeHODIJPQYjGoMbHUlU8oSxQVhwXvZCfpyNyTxK4pPMFEVKb2QXJ6XxOeHFuomPAKqr3q2eFB39jt2H5g2FJgHonsD+wZrMy9e4dne6rXEPU8gXDq3XpG1G6LYpglHGsIm2shtgIhVE5ULumWil9p6pAVVAhroKK79Rnxtk9VQ2emvPmvPpkToUkfN0Z9anzX69kg1Fp7IHhpFkuy6TbGNZGkpHZizjW5HGBM+pu+Rcyw5PFfZ4nyfMbE7N/hxOxeYNTeTMV5R54TlLCcQJZzjMmykVe5Az9oSqb4TLZ2itUfuxvacqLscOssrZCqN5y3Y1VNVJ8Klpt9anpX4vCmm34r6KmFgl5vblupvaXeVEcFYpGgt+FhZSZODs4EBLH39g94XcJexjEbHmAD07enJ6+Pz19c/D2/buj47eriVeDCMkyYxzzx5XxctucYV6U/Kp7a+3cFE79q2zZqgmEflru80rQRU5nX8/OXPKgvpXSL4je3t8YTctAlp5o1fst6rZ7uRfztrEhvsYJnWFJXCbdPEk8jpaZfNzbRMnSzIv32ttVL7BY6JOnhnbVav4bVLdrdNmz/r/obTXA5WLUfynK7vUpmHD4B/wBzZX3tKvfhSHTp4YJm4NutwVVLZH+usSP9WmiIKmgkt4TmGGJWzoaS0g7Bn0KZ9CPi/lqt5C8OLT/Sz2d8FdlqpZZNdKbuvg+/e231g/F/hCMOamDvWkXKBecPUA/aj1ob0v0HsgIEkKybfFdgdptXblzKRoCRauXr4SO9qzW9lwOb2n4/LXN6REV8oUKnm0vaSrb6pOEPZBZJVkcOxzutxFPV4MiZqdMe2tnwtFWdqupT1ZT/4cmyJZSo2TBz1PdEzTCW4cJBoPBiqx/2ShTP9bEG1lBiD46oe0NkT6ToOnTqKzzselSw0KxMMgM8lTkcUyEuMuT5LE48KtfCoOWzd5WBhtV9DqorP7E5enBhvW3VifhBa43Hd+07J+sIv3Xn92ao+8LnAvVafGSULpIb5itdp9lCUG4NNSOFekK/vJl2nUUrPSpyvjiJP/8GfDmVqq002ql/mzsai1rF5+bdquq2y/vcD7kNJm5+fKW8NJ4mcf1lkA1EeWZwCP0Lq9G47MvgXce3lg++lIfUnyp7oPc8Iudc9VSlJdQvf1B+YO2s82J6qKr7I9uGz6ZW5zVkuKBSn1G2UA0Y9g7evfu7UkPflSD6h39fvi+11gKvePDkze/NyUSKeLjw+PjttTpyfuW1PHJZUviRz/SpUpLvu+Boeqf+i0h0Ds5PHn/rvezAZdEyOPD49OmivKw9oWnWx1X6pte7dlyCCYZ3NF0pjcv5b0W6FhCqoN51pqM9UOxraSa4FyQ/7xJ3ST1Jyd2k4r/9Il1UiFxkhir+1PjAxZkZnipsbp2Nprz3Ui8fyLp9lcW6iu8PwV3nQ80HVL+S1CbpXe/hFUx+GVg685+p6wTu+AU93wZFvL16l6uGR59OQiqIMPUH+kX1cvzy03X81lx4z3QytF3vMwScga9anOLY2EsqSLSQF9zk9mcDFIiG9fg1VvtB/eHg8PBm7ebr8qLZ1vuyJXtkNVvEizzRNIsIRtu7sVreCDA0uQRBCnOVkg6yxhNJahObFNA/h1jK3y8aZh+GkO4VeQH1ny9vv22xOb5Iqpp0z1f76AHgjVeqjjTZgGOBlCWS7glC3xPGS/uiYsQVJNdSh8PVBLRh58bvKQpSDafJ0T3UPoNifJ8BxiHBZN39PtO2e2244jSmbihcrHXO+jVLUnzZGcl7Y8UDdv48tAzXn/bwnzm1RANU+7sfS6j+fXpCtkf2GyZYU5C1hD7hVdh9nVWb90od7i85QWSGtqIztVTUjwdhoraXB+QcZALnKopIf+T4+K+4ReGo/gku96F2TC66r/ElB1ie1vwKyNKiBDFcP6VIYii0/+VIWyoIir3tn/4bYOenQL8tNKB0Qav6TLqktOoPvrXspo0KwsYZaWoS8bOz/8LAAD//1BLBwjK1HOl/A8AAKQ0AABQSwMEFAAIAAgAAAAAAAAAAAAAAAAAAAAAABQAAAB3aW5kb3dzL3NlbmRsb2dzLnBzMaxTT0/bThC976eY35JDIv3s2EFEhdYtNKVglT8Rhkot5bCNJ/aK9a61O3ZbIb57tTYhUEDqoXvyzLx5M29m/GaDAQCE2ZeT03mWZp3l30WtjMgdCFCmgO+NzhUCGaASoTSOYGksWCQrsRUKWingoEFHn4+PTOFC1tN+2M9mZ+n8PD09+bfM/p2La/Q0tbCiQkILZgkCvqZzWEqFoEXVETddwf/hRykXJUgHDjWtKh4aR3sFarqn9QV9ZNxWXeDIFIA6r43UFLKNt+xyVuUK6b3UudTFcHTFug6GHcOlIyt1cdUZg7mgko0Yk0sY/jc8R0eBd/WB0QhuOphFaqxmt4wNDoxQGQnCPVs4SGC3R/BjpNLkPOEHSPx177uwkie8JKp3xuN4+iqcbobxZDuMp+NKLEqpcfxuYao6KYxQzpOuMg9R5GgdT3Zv+M+gckGL1kmjecInUTwJ4jjYjPgtu33QECQwGKa6NdcYnKGjviPYfdTxKLw3GRv0m35Jybz5CyU7m5OtyfTBKlYpqf4oFfKkm+RTWffbfEbfVhBtB1F8x7QGLZRETcHCWItKkDRa+j5fwvnz4gnf+5TNsv25agqpX4Kuy0dhHEZPYUaTkBqtL7geeThb+ddfaf5nclsJPxtliiDHWplfFWryRMNnmc6MwlQ7EnqBR9LRI4fHLWXR2E7+nXXi/6PA1UoS8G8hH11G/Xn7A3nmItZr/x0AAP//UEsHCIVf41EhAgAAXgQAAFBLAQIUABQACAAIAAAAAADK1HOl/A8AAKQ0AAAcAAAAAAAAAAAAAAAAAAAAAAB3aW5kb3dzL3dpbmRvd3Njc2VoZWxwZXIucHMxUEsBAhQAFAAIAAgAAAAAAIVf41EhAgAAXgQAABQAAAAAAAAAAAAAAAAARhAAAHdpbmRvd3Mvc2VuZGxvZ3MucHMxUEsFBgAAAAACAAIAjAAAAKkSAAAAAA==" +$zippedFiles = "UEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAcAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMdRb63ObSrL/7r+ii6g2dp0gv+LkRKe4dQjCNmsJVIDsZOMUi2EkzQYxujODHW02//utGd56ECe798P6gyxDv6b7N909D78Af4EZsIjiFQfMIGMoBk4gRjOcIngIGY4g4ziBWZZGHJOUHbwAS9I+IM4RbVBXJIBT4AtUyGWQpTGiwHg4x+n8OGLo+AmnMXli/YODFzDMucMkAfQVc4hIjKSIu5wIDM886M0T8hAmgzvLHjp3XmB4ZmC6ruMGU/vGdu5s7RRewCWhkKXo6wpFHMWAKCUUojCbLzg8rKVNUcijBTwkJPoidHzJHhBNEUesMIkhnq36K3baoXLo3NkjRx8Gl9bIDO4s/zpwTd/9qJ11MFn2rXNjBuYH05j6+vuRqZ13UEvJtuMH5gfL87XXHaTGtWncBPrECjzTvTXdwHBs2zR869byP2oXHZwTfeqZgTXWr5q63nRwXJl+4E3f26YfTFzz0vqgve2ktk1X983Ad25MO7h03EB3x9rvHSy26d857k1g2b7pXuqG6TUse/cMRn2oT3zTbXCdnnSwjXVbvzLHpu0H1qTJ1BV/Qx9ZhiPdbRlmoBuGM7X9JncXEAzH9nXLNt2h5LBsz9dHI3OonXYBYoPLndq2ZV9pp13IcCam7XnXm2q6IFGyXFqueaePRpLXcOxL62rqCuYudFj2rT6yhsFEd/WxKaJg2YH+t6lrFiK00y642E4wdIwb0w18J3g/tUbDAqHV2LXTLuwIcBp6YJiub11ahu6bnnbahZlqHm9ynXVBxhxP/I/bLF2AMW0x4QPPNIQv/JGnnXUh5Grs6YH5YaLbw0B3jWvr1tTOusCRM+RKJs6d6XrX5mgUTFzr1hqZV6Z21gUTye2ZfuCaV5bnux+DiemOLc+zHFs760LLNuetPpoKb3QmEcFljSeO6weGcWXemrbvaWedmaTNot/cTiaj6ZVll8ydOcXxg0tnag/bs10768wnFVOORHs6fm+62nkXMmqeievcWsJ/ln0VeIZrTXxPO++CiOfrrkghQzNwTeHSnCvwde9GO+9CS41izwwmunGjX5naeRdcKo6b6XvTtU3f9GrGLqTUqmyr5uhCSMVxbXvB2BlORd3rAkdjTsokW2npgkfFJHFSsXSBouE1S4Trw8ea73k5o07IJePrLnCImPrGJBh+tPWxZQQSy65uC74uYOTwKxLjZj583QWMyVTkgLrGa6+7QJHr8fWrNktn51G74L1l6+7HslvpQkTpB/ODMZoOzZYfuoAh+KbD3f7rwkbJt0tfd9YIjKnnO+NgqPu6GKD2+h28ABfxjIr+FrOywSQxEr1kxFC0jEXvCE8LlIIxuNf/mVE0DHl4b2SMk6X42n/AKcQEMUgJFy0v4z8ww5M5YTryg5FzpV2c/BtWeKaLWJbwfkLmzzfCcCYfhe7AcEYj0Vo6dpXVLrrA65qe9TczcLxg6IoqdtFZ9ibTnMwtuxVdarrULdG3XHRWwD28vjU2namvXXQWwD3Mt+NAmi/yujedCOQIOzor4h5RUzdvpTzT1y46i+NkGng308CyL526oGgXndVxj9J6naJbo6lrahed7VNTSt7IedaVrfuSsysn7tNvfjDMifimvenKjD/ymfnB1N6cHoi1ou345gBuqkXbLaIMk8Z0ikjKQ5yC8qhUGsc4xcsw2eK6w3wxCjli3Mi5EI1BA+W0f/Z7/0SBF3CHkwSiRZjOEWAulrpyCUkoRRGHx0K5nGZPCFi2WhHKIUVP1do1qiUX5JVZHg8fElSrnoTRl3COhAWPp/03/bNTNRQTt396/IDTkGLEjmtx6gaNWqxf1XAZv3nd5yHtz/+pHIgVPoJEjrLLmE0/tI1523/3I1saJHtMqXSZjyjlbETmc5zOh5gKJcbgvnCZTFb3IzJn95Mkm+OU3Y9xRAkjM943yHKVcdTPs6kn9xfMrxylYjD3ueD7OvJ+yL7Y4VIOo/3Ux0vk8XC52niFGG8ZVadO5eCg2C6Z4YQjChQtyaOAXZaDhIYRR5TB4f3JETwtcLSAkCKIwhXPKIpFbk4ZW/bRVwQk46uM58hJyHyOYuALSrL5AlbkCVG2QElyUChypSI7SxIG36AXgErRKgkjBC/vT16+gpcv4ftBSSwGxuTAvoHSO7xCXB2GHIF6Segy5ECOBtALFMlR7NfAHcUcqSMyP+wtEWPhHB3BtwMAgN6SzUGD8jH8q5Yv3+ecTj4aQXzQlDskT2lCwvgSJ8h5RPSa81UheBLScHkov4qfT/JvxBE9HIdpHHJC10Ivpxk6+vyJcYrT+eeKvDelyatfZh7KIIfCxEnIF88WhFPekGJ+xdwgMZJPjg7krxdwiamYaAsUfRHpgiEEeAahwAyCJ8wX+caYwGQqPjCDMKEojNcQhdECxUDyvbPb62EeAcFZYPiT5fSFxZ8HgyvEL4sXh8IbhQE9hkIaLUCDPw+P5BM8g0OBalUwQolzQ+gaYlrGuc0r9AyxyHGErmtl7HCT/VVt3ivJ5UkRzkr49vNgoCdJKQcjlhv0/aCyq9DYN0iWclBTBCdNgypYgjJlOJ2XHirzLpk1vKOCQVZrQSVdPaNkCb1Sw6eTz0ciHJuhVypdglm1OFpC4aiKEdQG05YEOa8iVIxMfKKEobZXo4yiCSWcRCRhjdDI1zjdRfDJWzOOln0b8b4n3mO+Lin89Qp9HgxyiiGahVnCX8FzOFhyXuBE/MwIRWG0gMPeqqATKaqUY6bZMg/9bZhkiP3YpKNm7KAM8fb41JTwonYwqHRvMu/03W9azdCi/n6w/a1tMH3EEZoQnPJxmIZzRIVDNoYh5vqGytpdPZLEE0rmFDE2oWiGKEojMSt7209rpp0cLz2coJQna1FvcZqhlw01cZE0RaIVVagcxhCH85QwjiPW9zhZPYU8EpnA4yHlNnpqgIrT9YY3eyGdS2R9m1KsiYTxB4wRX5BYU64QV/4AJ5NzXNsE+PeWHBdxulYNslyGaQzVF8VKH8kXpLqI8VysAqouVOaKVcGHEYOL/Ot6iJJw7aGIpDGD05M6esX+fNt6D3G1TLdQf+s1nokl0LgoUspliJO84JbOFHlBjLoPklKWwB2QaTtfuvmwMWMkoIsUqK9WVsrwfMGZkWBU5K+eaAY2odxDoi8BDWz0pDoP/xC9o1L3NPpqleBIOryU2BcdhwCH6ClYX/Y1PkrQUvhO2SG8X3Y5IoZl2d1JOKFkhSjHiH1SygqifBYwLlPpLq6xiF/EPinDjEpLxyxnajvMTMIVQ/EYJwlmeXTb0vY5r+/TMPoix3mYqzxqhOcH02n3zNxVRkrPoDivEgIS++vC91Yz0wThwbObGK27eejuPbS9LUwT7/Jp7q7GUD3EoZofYojVH2LCoq+Y13OhKS0fetWqF0xau92pXjcYtW2j5HFezdn05oQwviJpnjJCyovGnkLhWtH/qz5RJSRArRp5Rb/x+sVKwfDMfilnQ4ySs1QZQSZJ4YaIItENhykI5RwvEfCQyV6N5iLy7mtc+CHMzc3nrifajyxBsZCt529U8yuKMo5AqRt30eLnCTBbCvOVOlP+XSksVasR5x3E35VC44riNMKrMNmldFK9VKcMUSsG76Pnm2MQQSepqMNQ1Do9ivK2ys3SEXpECVzj+QIxLtW8AE7xfI5ovoUkfUDKWdMr3+UW/JU8+MUDVecA1aLiqK/HcZHFD08vjkTaQiITgOpUsuThMN7nRVALP5aeVushNlyhlgZUpomuTC7/BFPpVIhKp4Yz8bmi5BGLXlHk/2ZcXTTHjCO6aUyNswINalQjykpXGS/yd2NYmzPPkBCLn4Wx5pTI+dSq8f6FJLOVJy6zJPnxGkfrzcKE7V4rldZMWbEdMEcpomECq4yuCCvyRZ5+5IcokqpNeGvZUdpxtKe/z8cOcamt5hBt6qYZdWETeMq7dvEp4V+RwoZu+J+8POe5fbNPb1aJ51mR75qyRq14AQvOV2xwfMx4GH0hj4jOEvLUj8jyODw+f31x8e7i4vXxm3dvT8/e1IEXg/DRckVoSNe18mIFvgpp3j2UjWBrESj4xF9F91cGEHppsWQsmK4yHH8eDGz0JL4V1M/w3uFfCU4LRxaWSNFHLei228Jn47axtr4NExyHHNmE21mSONRcrvj6cBckCzXPXrbvF70I2UJuwjWki671PyC6Xe6L9vf/RW6rly4mo/wlIHvYw6DByR/wBzRn3vYC4QUMidxATcgcZOfOsOiu5NdluK42VhlKGeb4EUEc8rAlozGFpGHQwzCAXpTHq92N0vz84i9VOOFPoaqiqXvyXQuCHv7tt9aDfKkJ6hxVzt61oOQLSp6gF7RetFc4slNQvQSh1T7/1kztDrFYBOUNgYDV82dCR6dXSftRDm9J+PS5jekRZvyZAn7YqeKUt8UnCXlCcUmZ72CcHLU5tmeDAGYnTXuVqMHpXnSL0Cd16P+QANlTagQtuFkqe4KGeys3Qb/fr8H6l5001WsJvJHu+eYHyzecoSm3N3C67ZVNPDZNamjIJwaKIUtZFkWIsVmWJOt877C6HwctncpeBKul9zqgLH6iYiNix/zbqJPwDNObhu+a9luzSP762VW++XURZkx0WrQAlCzSO6LV7rN0xhDlqlj8mrKCP3+adu0qC3miMj47yf94O3l3K1XoabVSP+u7SsrGGfCuha/o9osV1/sMJ7GdLR8QLZQXeVwuCUQTUWwvrEG5vhmNB/eec+nf6a5Zn+GU5zxg+/dGRkVLUZzHKUf94oHUs8+I8syv6I8eGjZpe4yVlOwJc7nd2eBo+lA5ffv2zbkC38pBKae/n7xTGlNBOTs5f/17kyLhLDo7OTtrU12cv2tRnZ1ftyi+9QJZqiTlOwVUUf/EswSBcn5y/u6t8r3BzhHjZydnF00Rxb7vMzfKOm4X7LrltGc/jROY4TSWi5fiiA+kLyGVzhy0grG5v7YXVJMwY+i/L6i7qH4ysLtE/LcH1koZD5NErY+S1fchQ7HqpGp9Aq82491IvD+RdHu1huo08KfYbes9ToeY/hKrQdLZL/EKH/wyY+v6wkFRJ567VVZEpnabjIuTbgneu3mGcwlQiRjCg5ABpHmRvjyf60MrQoP68LEVuO3bHDWlePW+uG9QGieG/AKs/JR0FTL+ardgebQKogeBqTuS/6ZQ7P7uupyxyu879KVw82u4XCVoAEq5ng8jpi6xmDt9eckBxXPUTxFvXIIo/6fh+PGkf9J//Wb3RYn83Z4bEkK3T6p7JMss4XiVoB33NtgreEJA0mQNDOXbSSiNVwSnHETzucsh/4mx5TbeNVRv+7ALE/mNh93xQqJPlW2ucqwAI40rNQOpFuC0D0WHAA9oET5iQvNT9twFZbAL6rO+yJtcnhxvW4lT4GQ+T5BsG+X9mGJLCwiFBeEz/PWgaPDbfjTTmN1hvjhUjpWqC2tuZtXU7kjAsM1f7PNGm3dttB9cDJJswpzDT4U3P28nhSN5RSakyCcNsl+4CHUkC1nrPL7D5D3XhyrWhndutkGxPQzhtbncE6TAF2EqQoL+Nwvz05pfGI7AE++6CbVjdOU/RBVNcXsl9CsjShBj+XD+nSGwfHHzK0PYUThFuWk/+G2HnIOc+ZdLzU+UlO2ysTV7tnsMUNtj2BiSWhX7Rt2XT4s63qzpoBY1uirWrQan7YP/yNKx9OOzl447JWwfB+YnqfLwR1OsdCZvgGGShkUOap5n193l5jWclhTZP5Zoaihh5Vmy9qk8BHJmM4b458FgyiObPPV9Mk3xV/GmeT5cbEX3ojyvi9fajmtrynq9XqvjsRrHcH09WC4HjPVns5nSOHnoJSHj7St/rcfl2TVocLI1/valQOEERWl5YUP45hXDbcKGut6h3OPHS8RWYQpqDv89qk2xZmi4o3VtasfNxl7LhH2XHJsSDzYr1pZUtQWpwo/NGazBIYoWpP3wX2LqPiLKfaL+lZG0CG1xeVA8AQ3+zKU1lmjXhHF5DWEASg+lj4PyzC0VD+tJoYwaIZDEzZjsIixD0CKuHjYYiq6iOebB9qz6fqD8uWtIhSuaD3e7Imf+ByOpJ2fwTndUNyyl1bvC2bTcWaF8OFbcpG8+blCXTfQAlNP+6UnzVcuxG3ho0plVNsjjVf/ZIGo4seGVthNzpzWc8S+5zBWZWx4zfdtzXfh771s75Xzv/6MUXl/j+1n2mqG+A7xP0/f/CwAA//9QSwcI5mmr8jESAAA4PQAAUEsDBBQACAAIAAAAAAAAAAAAAAAAAAAAAAAUAAAAd2luZG93cy9zZW5kbG9ncy5wczGsU09P204Qve+nmN+SQyL97NhBRIXWLTSlYJU/EYZKLeWwjSf2ivWutTt2WyG+e7U2IVBA6qF78sy8eTNvZvxmgwEAhNmXk9N5lmad5d9FrYzIHQhQpoDvjc4VAhmgEqE0jmBpLFgkK7EVClop4KBBR5+Pj0zhQtbTftjPZmfp/Dw9Pfm3zP6di2v0NLWwokJCC2YJAr6mc1hKhaBF1RE3XcH/4UcpFyVIBw41rSoeGkd7BWq6p/UFfWTcVl3gyBSAOq+N1BSyjbfsclblCum91LnUxXB0xboOhh3DpSMrdXHVGYO5oJKNGJNLGP43PEdHgXf1gdEIbjqYRWqsZreMDQ6MUBkJwj1bOEhgt0fwY6TS5DzhB0j8de+7sJInvCSqd8bjePoqnG6G8WQ7jKfjSixKqXH8bmGqOimMUM6TrjIPUeRoHU92b/jPoHJBi9ZJo3nCJ1E8CeI42Iz4Lbt90BAkMBimujXXGJyho74j2H3U8Si8Nxkb9Jt+Scm8+QslO5uTrcn0wSpWKan+KBXypJvkU1n323xG31YQbQdRfMe0Bi2URE3BwliLSpA0Wvo+X8L58+IJ3/uUzbL9uWoKqV+CrstHYRxGT2FGk5AarS+4Hnk4W/nXX2n+Z3JbCT8bZYogx1qZXxVq8kTDZ5nOjMJUOxJ6gUfS0SOHxy1l0dhO/p114v+jwNVKEvBvIR9dRv15+wN55iLWa/8dAAD//1BLBwiFX+NRIQIAAF4EAABQSwECFAAUAAgACAAAAAAA5mmr8jESAAA4PQAAHAAAAAAAAAAAAAAAAAAAAAAAd2luZG93cy93aW5kb3dzY3NlaGVscGVyLnBzMVBLAQIUABQACAAIAAAAAACFX+NRIQIAAF4EAAAUAAAAAAAAAAAAAAAAAHsSAAB3aW5kb3dzL3NlbmRsb2dzLnBzMVBLBQYAAAAAAgACAIwAAADeFAAAAAA=" $global:KubeClusterConfigPath = "c:\k\kubeclusterconfig.json" $fipsEnabled = [System.Convert]::ToBoolean("false") @@ -212,9 +212,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -237,13 +239,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -257,22 +260,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("true") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -283,25 +282,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig ` -KubeDir $global:KubeDir ` -AADClientId $AADClientId ` @@ -328,22 +325,20 @@ try # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir ` -MasterFQDNPrefix $MasterFQDNPrefix ` -MasterIP $MasterIP ` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -358,9 +353,8 @@ try -MasterIP $MasterIP ` -AgentKey $AgentKey ` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -369,12 +363,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` -AzureCNIBinDir $global:AzureCNIBinDir ` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` -KubeDnsSearchPath $global:KubeDnsSearchPath ` -KubeClusterCIDR $global:KubeClusterCIDR ` @@ -382,7 +375,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` -TenantId $global:TenantId ` @@ -397,32 +390,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices ` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -434,19 +420,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -460,8 +444,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -499,6 +483,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() diff --git a/pkg/templates/templates_generated.go b/pkg/templates/templates_generated.go index a8e079428cd..0866320348a 100644 --- a/pkg/templates/templates_generated.go +++ b/pkg/templates/templates_generated.go @@ -8222,9 +8222,11 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\windows\windowscsehelper.ps1 # util functions only can be used after this line, for example, Write-Log +$global:OperationId = New-Guid + try { - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" + Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" # Exit early if the script has been executed if (Test-Path -Path $CSEResultFilePath -PathType Leaf) { @@ -8247,13 +8249,14 @@ try $global:CSEScriptsPackageUrl = $global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage Write-Log "CSEScriptsPackageUrl is set to $global:CSEScriptsPackageUrl" } + # Download CSE function scripts - Write-Log "Getting CSE scripts" + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl" $tempfile = 'c:\csescripts.zip' DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" Remove-Item -Path $tempfile -Force - + # Dot-source cse scripts with functions that are called in this script . c:\AzureData\windows\azurecnifunc.ps1 . c:\AzureData\windows\calicofunc.ps1 @@ -8267,22 +8270,18 @@ try $sshEnabled = [System.Convert]::ToBoolean("{{ WindowsSSHEnabled }}") if ( $sshEnabled ) { - Write-Log "Install OpenSSH" Install-OpenSSH -SSHKeys $SSHKeys } - Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID - Write-Log "Resize os drive if possible" Resize-OSDrive - - Write-Log "Initialize data disks" + Initialize-DataDisks - - Write-Log "Create required data directories as needed" + Initialize-DataDirectories - + + Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts" Create-Directory -FullPath "c:\k" Write-Log "Remove `+"`"+`"NT AUTHORITY\Authenticated Users`+"`"+`" write permissions on files in c:\k" icacls.exe "c:\k" /inheritance:r @@ -8293,25 +8292,23 @@ try icacls.exe "c:\k" Get-ProvisioningScripts Get-LogCollectionScripts - + Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp - - Write-Log "Download kubelet binaries and unzip" + Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL - - Write-Log "Installing ContainerD" + $cniBinPath = $global:AzureCNIBinDir $cniConfigPath = $global:AzureCNIConfDir if ($global:NetworkPlugin -eq "kubenet") { $cniBinPath = $global:CNIPath $cniConfigPath = $global:CNIConfigPath } - Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion + Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment - + # For AKSClustomCloud, TargetEnvironment must be set to AzureStackCloud - Write-Log "Write Azure cloud provider config" Write-AzureConfig `+"`"+` -KubeDir $global:KubeDir `+"`"+` -AADClientId $AADClientId `+"`"+` @@ -8344,22 +8341,20 @@ try Get-CACertificates {{end}} - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate `+"`"+` -KubeDir $global:KubeDir - + if ($global:EnableCsiProxy) { New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir } if ($global:TLSBootstrapToken) { - Write-Log "Write TLS bootstrap kubeconfig" Write-BootstrapKubeConfig -CACertificate $global:CACertificate `+"`"+` -KubeDir $global:KubeDir `+"`"+` -MasterFQDNPrefix $MasterFQDNPrefix `+"`"+` -MasterIP $MasterIP `+"`"+` -TLSBootstrapToken $global:TLSBootstrapToken - + # NOTE: we need kubeconfig to setup calico even if TLS bootstrapping is enabled # This kubeconfig will deleted after calico installation. # TODO(hbc): once TLS bootstrap is fully enabled, remove this if block @@ -8374,9 +8369,8 @@ try -MasterIP $MasterIP `+"`"+` -AgentKey $AgentKey `+"`"+` -AgentCertificate $global:AgentCertificate - + if ($global:EnableHostsConfigAgent) { - Write-Log "Starting hosts config agent" New-HostsConfigService } @@ -8385,12 +8379,11 @@ try # Configure network policy. Get-HnsPsm1 -HNSModule $global:HNSModule Import-Module $global:HNSModule - - Write-Log "Installing Azure VNet plugins" + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir `+"`"+` -AzureCNIBinDir $global:AzureCNIBinDir `+"`"+` -VNetCNIPluginsURL $global:VNetCNIPluginsURL - + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir `+"`"+` -KubeDnsSearchPath $global:KubeDnsSearchPath `+"`"+` -KubeClusterCIDR $global:KubeClusterCIDR `+"`"+` @@ -8398,7 +8391,7 @@ try -VNetCIDR $global:VNetCIDR `+"`"+` -IsDualStackEnabled $global:IsDualStackEnabled `+"`"+` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled - + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig `+"`"+` -TenantId $global:TenantId `+"`"+` @@ -8413,32 +8406,25 @@ try } New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled - + Install-KubernetesServices `+"`"+` -KubeDir $global:KubeDir - Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer - - Write-Log "Adjust pagefile size" Adjust-PageFileSize - - Write-Log "Start preProvisioning script" + Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script" PREPROVISION_EXTENSION - - Write-Log "Update service failure actions" Update-ServiceFailureActions Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask Update-DefenderPreferences - $windowsVersion = Get-WindowsVersion if ($windowsVersion -ne "1809") { - Write-Log "Skip secure TLS protocols for Windows version: $windowsVersion" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion" } else { - Write-Log "Enable secure TLS protocols" + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols" try { . C:\k\windowssecuretls.ps1 Enable-SecureTls @@ -8450,19 +8436,17 @@ try Enable-FIPSMode -FipsEnabled $fipsEnabled if ($global:WindowsGmsaPackageUrl) { - Write-Log "Start to install Windows gmsa package" Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl } Check-APIServerConnectivity -MasterIP $MasterIP if ($global:WindowsCalicoPackageURL) { - Write-Log "Start calico installation" Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp } Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL - + if (Test-Path $CacheDir) { Write-Log "Removing aks cache directory" @@ -8476,8 +8460,8 @@ try } Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes - - Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot" + + Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Winodws node without reboot" Start-ScheduledTask -TaskName "k8s-restart-job" $timeout = 180 ## seconds @@ -8515,6 +8499,8 @@ finally Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode" echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8 + Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." + # Flush stdout to C:\AzureData\CustomDataSetupScript.log [Console]::Out.Flush() @@ -8667,6 +8653,10 @@ $global:StableContainerdPackage = "v1.6.21-azure.1/binaries/containerd-v1.6.21-a # The latest containerd version $global:LatestContainerdPackage = "v1.7.9-azure.1/binaries/containerd-v1.7.9-azure.1-windows-amd64.tar.gz" +$global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\" +$global:TaskName = "" +$global:TaskTimeStamp = "" +$global:TestingDir = "C:\AzureData\" # This filter removes null characters (\0) which are captured in nssm.exe output when logged through powershell filter RemoveNulls { $_ -replace '\0', '' } @@ -8752,7 +8742,7 @@ function Set-ExitCode function Postpone-RestartComputer { - Write-Log "Creating an one-time task to restart the VM" + Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument " -Command `+"`"+`"Restart-Computer -Force`+"`"+`"" $principal = New-ScheduledTaskPrincipal -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest # trigger this task once @@ -8904,6 +8894,8 @@ function Install-Containerd-Based-On-Kubernetes-Version { $KubernetesVersion ) + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerdBasedOnKubernetesVersion" -TaskMessage "Start to install ContainerD based on kubernetes version. ContainerdUrl: $global:ContainerdUrl, KubernetesVersion: $global:KubeBinariesVersion" + # In the past, $global:ContainerdUrl is a full URL to download Windows containerd package. # Example: "https://acs-mirror.azureedge.net/containerd/windows/v0.0.46/binaries/containerd-v0.0.46-windows-amd64.tar.gz" # To support multiple containerd versions, we only set the endpoint in $global:ContainerdUrl. @@ -8922,7 +8914,59 @@ function Install-Containerd-Based-On-Kubernetes-Version { } $ContainerdUrl = $ContainerdUrl + $containerdPackage } + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallContainerd" -TaskMessage "Start to install ContainerD. ContainerdUrl: $ContainerdUrl" Install-Containerd -ContainerdUrl $ContainerdUrl -CNIBinDir $CNIBinDir -CNIConfDir $CNIConfDir -KubeDir $KubeDir +} + +function Logs-To-Event { + Param( + [Parameter(Mandatory = $true)][string] + $TaskName, + [Parameter(Mandatory = $true)][string] + $TaskMessage + ) + $eventLevel="Informational" + if ($global:ExitCode -ne 0) { + $eventLevel="Error" + } + + $eventsFileName=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $currentTime=$(Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff") + + $lastTaskName = "" + $lastTaskDuration = 0 + if ($global:TaskTimeStamp -ne "") { + $lastTaskName = $global:TaskName + $lastTaskDuration = $(New-Timespan -Start $global:TaskTimeStamp -End $currentTime) + } + + $global:TaskName = $TaskName + $global:TaskTimeStamp = $currentTime + + Write-Log "$global:TaskName - $TaskMessage" + $TaskMessage = (echo $TaskMessage | ConvertTo-Json) + $messageJson = @" + { + "HostName": "$env:computername", + "LastTaskName": "$lastTaskName", + "LastTaskDuration": "$lastTaskDuration", + "CurrentTaskMessage": $TaskMessage + } +"@ + $messageJson = (echo $messageJson | ConvertTo-Json) + + $jsonString = @" + { + "Timestamp": "$global:TaskTimeStamp", + "OperationId": "$global:OperationId", + "Version": "1.10", + "TaskName": "$global:TaskName", + "EventLevel": "$eventLevel", + "Message": $messageJson + } +"@ + echo $jsonString | Set-Content ${global:EventsLoggingDir}${eventsFileName}.json + Copy-Item ${global:EventsLoggingDir}${eventsFileName}.json ${global:TestingDir}${eventsFileName}.json }`) func windowsWindowscsehelperPs1Bytes() ([]byte, error) { diff --git a/staging/cse/windows/azurecnifunc.ps1 b/staging/cse/windows/azurecnifunc.ps1 index 6bc726fd6a9..9326eec956a 100644 --- a/staging/cse/windows/azurecnifunc.ps1 +++ b/staging/cse/windows/azurecnifunc.ps1 @@ -8,6 +8,8 @@ function Install-VnetPlugins [Parameter(Mandatory=$true)][string] $VNetCNIPluginsURL ) + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallVnetPlugins" -TaskMessage "Start to install Azure VNet plugins. VnetCNIPluginsURL: $global:VNetCNIPluginsURL" + # Create CNI directories. Create-Directory -FullPath $AzureCNIBinDir -DirectoryUsage "storing Azure CNI binaries" Create-Directory -FullPath $AzureCNIConfDir -DirectoryUsage "storing Azure CNI configuration" @@ -43,6 +45,8 @@ function Set-AzureCNIConfig [Parameter(Mandatory=$false)][bool] $IsAzureCNIOverlayEnabled ) + Logs-To-Event -TaskName "AKS.WindowsCSE.SetAzureCNIConfig" -TaskMessage "Start to set Azure CNI config. IsDualStackEnabled: $global:IsDualStackEnabled, IsAzureCNIOverlayEnabled: $global:IsAzureCNIOverlayEnabled, IsDisableWindowsOutboundNat: $global:IsDisableWindowsOutboundNat" + $fileName = [Io.path]::Combine("$AzureCNIConfDir", "10-azure.conflist") $configJson = Get-Content $fileName | ConvertFrom-Json $configJson.plugins.dns.Nameservers[0] = $KubeDnsServiceIp @@ -297,6 +301,7 @@ function GenerateAzureStackCNIConfig [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string] $KubeDir ) + Logs-To-Event -TaskName "AKS.WindowsCSE.GenerateAzureStackCNIConfig" -TaskMessage "Start to generate Azure Stack CNI config" $networkInterfacesFile = "$KubeDir\network-interfaces.json" $azureCNIConfigFile = "$KubeDir\interfaces.json" @@ -393,6 +398,7 @@ function New-ExternalHnsNetwork [Parameter(Mandatory=$true)][bool] $IsDualStackEnabled ) + Logs-To-Event -TaskName "AKS.WindowsCSE.NewExternalHnsNetwork" -TaskMessage "Start to create new external hns network" Write-Log "Creating new HNS network `"ext`"" $externalNetwork = "ext" @@ -469,6 +475,7 @@ function Get-HnsPsm1 [Parameter(Mandatory=$true)][string] $HNSModule ) + Logs-To-Event "ASK.WindowsCSE.GetAndImportHNSModule" -TaskMessage "Start to get and import hns module. NetworkPlugin: $global:NetworkPlugin" # HNSModule is C:\k\hns.v2.psm1 when container runtime is Containerd $fileName = [IO.Path]::GetFileName($HNSModule) diff --git a/staging/cse/windows/calicofunc.ps1 b/staging/cse/windows/calicofunc.ps1 index 850d0938669..a13e36ee1b4 100644 --- a/staging/cse/windows/calicofunc.ps1 +++ b/staging/cse/windows/calicofunc.ps1 @@ -96,6 +96,7 @@ function Start-InstallCalico { [parameter(Mandatory=$true)] $KubeDnsServiceIp, [parameter(Mandatory=$false)] $CalicoNs = "calico-system" ) + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallCalico" -TaskMessage "Start calico installation. WindowsCalicoPackageURL: $global:WindowsCalicoPackageURL" Write-Log "Download Calico" Get-CalicoPackage -RootDir $RootDir diff --git a/staging/cse/windows/configfunc.ps1 b/staging/cse/windows/configfunc.ps1 index d6cafdd667e..c719115f9de 100644 --- a/staging/cse/windows/configfunc.ps1 +++ b/staging/cse/windows/configfunc.ps1 @@ -7,6 +7,7 @@ function Set-TelemetrySetting [Parameter(Mandatory=$true)][string] $WindowsTelemetryGUID ) + Logs-To-Event -TaskName "AKS.WindowsCSE.SetTelemetrySetting" -TaskMessage "Start to apply telemetry data setting. WindowsTelemetryGUID: $global:WindowsTelemetryGUID" Set-ItemProperty -Path "HKLM:\Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection" -Name "CommercialId" -Value $WindowsTelemetryGUID -Force } @@ -14,6 +15,7 @@ function Set-TelemetrySetting # This approach was recommended by the Windows Storage team to avoid performance delay when calling Get-PartitionSupportedSize function Resize-OSDrive { + Logs-To-Event -TaskName "AKS.WindowsCSE.ResizeOSDrive" -TaskMessage "Start to resize os drive if possible" try { $osDrive = ((Get-WmiObject Win32_OperatingSystem -ErrorAction Stop).SystemDrive).TrimEnd(":") @@ -35,6 +37,7 @@ function Resize-OSDrive # https://docs.microsoft.com/en-us/powershell/module/storage/new-partition function Initialize-DataDisks { + Logs-To-Event -TaskName "AKS.WindowsCSE.InitializeDataDisks" -TaskMessage "Start to initialize data disks" Get-Disk | Where-Object PartitionStyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -UseMaximumSize -AssignDriveLetter | Format-Volume -FileSystem NTFS -Force } @@ -43,6 +46,8 @@ function Initialize-DataDisks # (This only affects installations with UI) function Set-Explorer { + Logs-To-Event -TaskName "AKS.WindowsCSE.SetExplorer" -TaskMessage "Start to disable Internet Explorer compat mode and set homepage" + New-Item -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer" New-Item -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\BrowserEmulation" New-ItemProperty -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\BrowserEmulation" -Name IntranetCompatibilityMode -Value 0 -Type DWord @@ -53,11 +58,15 @@ function Set-Explorer # Pagefile adjustments function Adjust-PageFileSize() { + Logs-To-Event -TaskName "AKS.WindowsCSE.AdjustPageFileSize" -TaskMessage "Start to adjust pagefile size" + wmic pagefileset set InitialSize=8096,MaximumSize=8096 } function Adjust-DynamicPortRange() { + Logs-To-Event -TaskName "AKS.WindowsCSE.AdjustDynamicPortRange" -TaskMessage "Start to adjust dynamic port range" + # Kube-proxy reserves 63 ports per service which limits clusters with Windows nodes # to ~225 services if default port reservations are used. # https://docs.microsoft.com/en-us/virtualization/windowscontainers/kubernetes/common-problems#load-balancers-are-plumbed-inconsistently-across-the-cluster-nodes @@ -80,6 +89,8 @@ function Adjust-DynamicPortRange() # Service start actions. These should be split up later and included in each install step function Update-ServiceFailureActions { + Logs-To-Event -TaskName "AKS.WindowsCSE.UpdateServiceFailureActions" -TaskMessage "Start to update service failure actions" + sc.exe failure "kubelet" actions= restart/60000/restart/60000/restart/60000 reset= 900 sc.exe failure "kubeproxy" actions= restart/60000/restart/60000/restart/60000 reset= 900 sc.exe failure "containerd" actions= restart/60000/restart/60000/restart/60000 reset= 900 @@ -174,6 +185,7 @@ function Install-GmsaPlugin { [Parameter(Mandatory=$true)] [String] $GmsaPackageUrl ) + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallGmsaPlugin" -TaskMessage "Start to install Windows gmsa package. WindowsGmsaPackageUrl: $global:WindowsGmsaPackageUrl" $tempInstallPackageFoler = [Io.path]::Combine($env:TEMP, "CCGAKVPlugin") $tempPluginZipFile = [Io.path]::Combine($ENV:TEMP, "gmsa.zip") @@ -272,6 +284,7 @@ function Install-OpenSSH { [Parameter(Mandatory = $true)][string[]] $SSHKeys ) + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallOpenSSH" -TaskMessage "Start to install OpenSSH" $adminpath = "c:\ProgramData\ssh" $adminfile = "administrators_authorized_keys" @@ -334,6 +347,8 @@ function New-CsiProxyService { $KubeDir ) + Logs-To-Event -TaskName "AKS.WindowsCSE.StartCsiProxyService" -TaskMessage "Start Csi proxy service. CsiProxyUrl: $global:CsiProxyUrl" + $tempdir = New-TemporaryDirectory $binaryPackage = "$tempdir\csiproxy.tar" @@ -363,6 +378,8 @@ function New-CsiProxyService { } function New-HostsConfigService { + Logs-To-Event -TaskName "AKS.WindowsCSE.StartHostConfigService" -TaskMessage "Start hosts config agent" + $HostsConfigParameters = [io.path]::Combine($KubeDir, "hostsconfigagent.ps1") & "$KubeDir\nssm.exe" install hosts-config-agent C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe | RemoveNulls @@ -403,6 +420,8 @@ function Enable-GuestVMLogs { [Parameter(Mandatory = $true)][int] $IntervalInMinutes ) + Logs-To-Event -TaskName "AKS.WindowsCSE.EnableGuestVMLogs" -TaskMessage "Start to enable Guest VM Logs. LogGeneratorIntervalInMinutes: $LogGeneratorIntervalInMinutes" + if ($IntervalInMinutes -le 0) { Write-Log "Do not add AKS logs in GuestVMLogs" return @@ -466,6 +485,8 @@ function Retag-ImagesForAzureChinaCloud { [Parameter(Mandatory=$true)][string] $TargetEnvironment ) + + Logs-To-Event -TaskName "AKS.WindowsCSE.RetagImagesForAzureChinaCloud" -TaskMessage "Start to retag images for Azure China Cloud" $isExist=$false $imageList=$(ctr.exe -n k8s.io image ls | select -Skip 1) diff --git a/staging/cse/windows/kubeletfunc.ps1 b/staging/cse/windows/kubeletfunc.ps1 index 6066bab3b80..e4915cb4b2b 100644 --- a/staging/cse/windows/kubeletfunc.ps1 +++ b/staging/cse/windows/kubeletfunc.ps1 @@ -44,6 +44,8 @@ function Write-AzureConfig { $UseContainerD = $false ) + Logs-To-Event -TaskName "AKS.WindowsCSE.WriteAzureCloudProviderConfig" -TaskMessage "Start to write Azure Cloud Provider Config" + if ( $VmType -eq "vmss" -And -Not $PrimaryAvailabilitySetName -And -Not $PrimaryScaleSetName ) { Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG -ErrorMessage "Either PrimaryAvailabilitySetName or PrimaryScaleSetName must be set" } @@ -85,6 +87,7 @@ function Write-CACert { [Parameter(Mandatory = $true)][string] $KubeDir ) + Logs-To-Event -TaskName "AKS.WindowsCSE.WriteCACert" -TaskMessage "Start to write ca root" $caFile = [io.path]::Combine($KubeDir, "ca.crt") [System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($CACertificate)) | Out-File -Encoding ascii $caFile } @@ -104,6 +107,8 @@ function Write-KubeConfig { [Parameter(Mandatory = $true)][string] $KubeDir ) + Logs-To-Event -TaskName "AKS.WindowsCSE.WriteKubeConfig" -TaskMessage "Start to write kube config" + $kubeConfigFile = [io.path]::Combine($KubeDir, "config") $kubeConfig = @" @@ -144,6 +149,8 @@ function Write-BootstrapKubeConfig { [Parameter(Mandatory = $true)][string] $KubeDir ) + Logs-To-Event -TaskName "AKS.WindowsCSE.WriteBootstrapKubeConfig" -TaskMessage "Start to write TLS bootstrap kubeconfig" + $bootstrapKubeConfigFile = [io.path]::Combine($KubeDir, "bootstrap-config") $bootstrapKubeConfig = @" @@ -175,6 +182,7 @@ function Get-KubePackage { [Parameter(Mandatory = $true)][string] $KubeBinariesSASURL ) + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadKubletBinaries" -TaskMessage "Start to download kubelet binaries and unzip. KubeBinariesPackageSASURL: $global:KubeBinariesPackageSASURL" $zipfile = "c:\k.zip" for ($i = 0; $i -le 10; $i++) { @@ -260,6 +268,7 @@ function Install-KubernetesServices { [Parameter(Mandatory = $true)][string] $KubeDir ) + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallKubernetesServices" -TaskMessage "Start to install kubernetes services" # TODO ksbrmnn fix callers to this function diff --git a/staging/cse/windows/kubernetesfunc.ps1 b/staging/cse/windows/kubernetesfunc.ps1 index 01b87b7fe43..08fc59953ba 100644 --- a/staging/cse/windows/kubernetesfunc.ps1 +++ b/staging/cse/windows/kubernetesfunc.ps1 @@ -29,6 +29,8 @@ function Initialize-DataDirectories { # Some of the Kubernetes tests that were designed for Linux try to mount /tmp into a pod # On Windows, Go translates to c:\tmp. If that path doesn't exist, then some node tests fail + Logs-To-Event -TaskName "AKS.WindowsCSE.InitializeDataDirectories" -TaskMessage "Start to create required data directories as needed" + $requiredPaths = 'c:\tmp' $requiredPaths | ForEach-Object { @@ -46,6 +48,7 @@ function Get-LogCollectionScripts { } function Register-LogsCleanupScriptTask { + Logs-To-Event -TaskName "AKS.WindowsCSE.RegisterLogsCleanupScriptTask" -TaskMessage "Start to register logs cleanup script task" Write-Log "Creating a scheduled task to run windowslogscleanup.ps1" $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-File `"c:\k\windowslogscleanup.ps1`"" @@ -56,6 +59,7 @@ function Register-LogsCleanupScriptTask { } function Register-NodeResetScriptTask { + Logs-To-Event -TaskName "AKS.WindowsCSE.RegisterNodeResetScriptTask" -TaskMessage "Start to register node reset script task. HNSRemediatorIntervalInMinutes: $global:HNSRemediatorIntervalInMinutes" Write-Log "Creating a startup task to run windowsnodereset.ps1" $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-File `"c:\k\windowsnodereset.ps1`"" @@ -74,6 +78,8 @@ function Write-KubeClusterConfig { $KubeDnsServiceIp ) + Logs-To-Event -TaskName "AKS.WindowsCSE.WriteKubeClusterConfig" -TaskMessage "Start to write KubeCluster Config. WindowsPauseImageURL: $global:WindowsPauseImageURL" + $Global:ClusterConfiguration = [PSCustomObject]@{ } $Global:ClusterConfiguration | Add-Member -MemberType NoteProperty -Name Cri -Value @{ @@ -133,6 +139,8 @@ function Write-KubeClusterConfig { } function Update-DefenderPreferences { + Logs-To-Event -TaskName "AKS.WindowsCSE.UpdateDefenderPreferences" -TaskMessage "Start to update defender preferences" + Add-MpPreference -ExclusionProcess "c:\k\kubelet.exe" Add-MpPreference -ExclusionProcess "c:\k\kube-proxy.exe" diff --git a/staging/cse/windows/nvidiagpudriverfunc.ps1 b/staging/cse/windows/nvidiagpudriverfunc.ps1 index 4373bf468d8..51c9b8aa5b5 100644 --- a/staging/cse/windows/nvidiagpudriverfunc.ps1 +++ b/staging/cse/windows/nvidiagpudriverfunc.ps1 @@ -6,6 +6,7 @@ function Start-InstallGPUDriver { [Parameter(Mandatory = $false)] [string]$GpuDriverURL ) + Logs-To-Event -TaskName "AKS.WindowsCSE.InstallGPUDriver" -TaskMessage "Start to install GPU driver. ConfigGPUDriverIfNeeded: $global:ConfigGPUDriverIfNeeded, GpuDriverURL: $global:GpuDriverURL" if (-not $EnableInstall) { Write-Log "ConfigGPUDriverIfNeeded is false. GPU driver installation skipped as per configuration."