Merge branch 'master' into andyzhangx/upgrade-csi-driver-image-versio…

…ns-linux

.pipelines/.vsts-vhd-builder-release-windows.yaml

@@ -1,5 +1,25 @@
 name: $(Date:yyyyMMdd)$(Rev:.r)_$(Build.SourceBranchName)_$(BuildID)
 trigger: none
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - .pipelines/.vsts-vhd-builder-release-windows.yaml
+    - .pipelines/templates/.builder-release-template-windows.yaml
+    - packer.mk
+    - vhdbuilder/packer/packer-plugin.pkr.hcl
+    - vhdbuilder/packer/*.ps1
+    - vhdbuilder/packer/test/*.ps1
+    - vhdbuilder/packer/test/run-test.sh
+    - vhdbuilder/packer/backfill-cleanup.sh
+    - vhdbuilder/packer/cleanup.sh
+    - vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh
+    - vhdbuilder/packer/generate-vhd-publishing-info.sh
+    - vhdbuilder/packer/init-variables.sh
+    - vhdbuilder/packer/windows-image.env
+    - vhdbuilder/packer/windows-vhd-builder-sig.json
 
 pool:
   name: $(AZURE_POOL_NAME)

.pipelines/e2e-windows.yaml

@@ -1,14 +1,14 @@
 name: $(Date:yyyyMMdd)$(Rev:.r)_e2e
 trigger: none
-
 pr:
   branches:
     include:
-    - main
+    - master
     - 'official/*'
   paths:
     include:
-    - '.pipelines/*'
+    - .pipelines/e2e-windows.yaml
+    - .pipelines/templates/e2e-windows-template.yaml
     - 'e2e/windows/*'
     - 'parts/windows/*'
     - 'staging/cse/windows/*'

.pipelines/templates/e2e-windows-template.yaml

@@ -21,8 +21,6 @@ parameters:
 jobs:
 - job: Setup_Test_Cluster
   steps:
-    - bash: pip install -Iv azure-cli==2.45.0 --extra-index-url https://azurecliprod.blob.core.windows.net/edge
-      displayName: use specified version of az cli
     - bash: |
         az login --identity
         az account set -s $(AZURE_E2E_SUBSCRIPTION_ID)
@@ -47,7 +45,8 @@ jobs:
           fi
         else
           if [[ -z "${KUBERNETES_VERSION}" ]]; then
-            KUBERNETES_VERSION=$(az aks get-versions -l $AZURE_BUILD_LOCATION --query "orchestrators[?default==\`true\`].orchestratorVersion" -otsv)
+            defaultMajorVersion=$(az aks get-versions -l $AZURE_BUILD_LOCATION --query "values[?isDefault==\`true\`].version" -otsv)
+            KUBERNETES_VERSION=$(az aks get-versions -l $AZURE_BUILD_LOCATION -o table | awk '{print $1}' | awk "/$defaultMajorVersion/ {print}" | sort -r | awk 'NR==1')
             echo "Using default kubernetes version ${KUBERNETES_VERSION} for the windows package"
           fi
           export windowsPackageURL="https://acs-mirror.azureedge.net/kubernetes/v${KUBERNETES_VERSION}/windowszip/v${KUBERNETES_VERSION}-1int.zip"
@@ -85,8 +84,6 @@ jobs:
   steps:
     - download: current
       artifact: ${{ parameters.windowsImage }}${{ parameters.windowsGpuDriverSuffix }}-clusterConfig
-    - bash: pip install -Iv azure-cli==2.45.0 --extra-index-url https://azurecliprod.blob.core.windows.net/edge
-      displayName: use specified version of az cli
     - bash: |
         az login --identity
         az account set -s $(AZURE_E2E_SUBSCRIPTION_ID)
@@ -111,7 +108,8 @@ jobs:
           export WINDOWS_PACKAGE_VERSION=$(echo $windowsPackageURL | grep -oP '(?<=/v)\d+\.\d+\.\d+' | head -1)
         else
           if [[ -z "${KUBERNETES_VERSION}" ]]; then
-            KUBERNETES_VERSION=$(az aks get-versions -l $AZURE_BUILD_LOCATION --query "orchestrators[?default==\`true\`].orchestratorVersion" -otsv)
+            defaultMajorVersion=$(az aks get-versions -l $AZURE_BUILD_LOCATION --query "values[?isDefault==\`true\`].version" -otsv)
+            KUBERNETES_VERSION=$(az aks get-versions -l $AZURE_BUILD_LOCATION -o table | awk '{print $1}' | awk "/$defaultMajorVersion/ {print}" | sort -r | awk 'NR==1')
             echo "Using default kubernetes version ${KUBERNETES_VERSION} for the windows package"
           fi
           export windowsPackageURL="https://acs-mirror.azureedge.net/kubernetes/v${KUBERNETES_VERSION}/windowszip/v${KUBERNETES_VERSION}-1int.zip"

e2e/windows/e2e-helper.sh

@@ -19,7 +19,7 @@ exec_on_host() {
 backfill_clean_storage_container() {
     set +x
     # Get supported kubernetes versions
-    versions=$(az aks get-versions --location $AZURE_BUILD_LOCATION --query "orchestrators[].orchestratorVersion" -o tsv)
+    versions=$(az aks get-versions --location $AZURE_BUILD_LOCATION -o table | tail -n +3 | awk '{print $1}')
     k8s_versions=${versions//./}
 
     # Get container names e.g. akswinstore2022-1256 (for this $container_version would be "1256")

packer.mk

@@ -95,8 +95,5 @@ generate-sas: az-login
 convert-sig-to-classic-storage-account-blob: az-login
 	@./vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh
 
-windows-vhd-publishing-info: az-login
-	@./vhdbuilder/packer/generate-windows-vhd-publishing-info.sh
-
 test-building-vhd: az-login
 	@./vhdbuilder/packer/test/run-test.sh

parts/windows/kuberneteswindowssetup.ps1

@@ -293,15 +293,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -468,7 +459,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -464,7 +455,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+K8S116/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+K8S117/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+K8S118/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+K8S119/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }
 

pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData

@@ -289,15 +289,6 @@ try
     Write-Log "Download kubelet binaries and unzip"
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
-    # This overwrites the binaries that are downloaded from the custom packge with binaries.
-    # The custom package has a few files that are necessary for future steps (nssm.exe)
-    # this is a temporary work around to get the binaries until we depreciate
-    # custom package and nssm.exe as defined in aks-engine#3851.
-    if ($global:WindowsKubeBinariesURL){
-        Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-        Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-    }
-
     Write-Log "Installing ContainerD"
     $cniBinPath = $global:AzureCNIBinDir
     $cniConfigPath = $global:AzureCNIConfDir
@@ -458,7 +449,7 @@ try
 
     if (Test-Path $CacheDir)
     {
-        Write-Log "Removing aks-engine bits cache directory"
+        Write-Log "Removing aks cache directory"
         Remove-Item $CacheDir -Recurse -Force
     }