Merge branch 'master' into dependabot/go_modules/github.com/gorilla/h…

…andlers-1.5.2
Azure · Apr 17, 2024 · 2859a5c · 2859a5c
2 parents 9c86229 + dbbc9fc
commit 2859a5c
Show file tree

Hide file tree

Showing 173 changed files with 1,557 additions and 175 deletions.
diff --git a/.github/workflows/golangci-lint-pr.yml b/.github/workflows/golangci-lint-pr.yml
@@ -20,7 +20,7 @@ jobs:
           go-version: '1.19'
       - uses: actions/checkout@v4
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: v1.51.2

diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
@@ -20,7 +20,7 @@ jobs:
           go-version: '1.19'
       - uses: actions/checkout@v4
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: v1.51.2

diff --git a/parts/linux/cloud-init/artifacts/manifest.json b/parts/linux/cloud-init/artifacts/manifest.json
@@ -40,10 +40,13 @@
         "versions": [
             "1.27.7",
             "1.27.9",
+            "1.27.12",
             "1.28.3",
             "1.28.5",
+            "1.28.8",
             "1.29.0",
-            "1.29.2"
+            "1.29.2",
+            "1.29.3"
         ]
     },
     "_template": {

diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1
@@ -164,6 +164,9 @@ $global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}"
 $global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}}
 $global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}}
 
+# Kubelet credential provider
+$global:CredentialProviderURL = "{{GetParameter "windowsCredentialProviderURL"}}"
+
 # CSI Proxy settings
 $global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}");
 $global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}";
@@ -292,7 +295,9 @@ try
     Get-LogCollectionScripts
 
     Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp
-
+
+    Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{ AKSCustomCloudContainerRegistryDNSSuffix }}"{{else}}""{{end}} 
+
     Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
 
     $cniBinPath = $global:AzureCNIBinDir

diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go
@@ -1642,6 +1642,50 @@ var _ = Describe("Assert generated customData and cseCmd for Windows", func() {
 					},
 				}
 			}),
+		Entry("AKSWindows2019 with out of tree credential provider", "AKSWindows2019+ootcredentialprovider", "1.29.0", func(config *datamodel.NodeBootstrappingConfiguration) {
+			config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = to.BoolPtr(true)
+			config.KubeletConfig["--image-credential-provider-config"] = "c:\\var\\lib\\kubelet\\credential-provider-config.yaml"
+			config.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider"
+		}),
+		Entry("AKSWindows2019 with custom cloud and out of tree credential provider", "AKSWindows2019+CustomCloud+ootcredentialprovider", "1.29.0",
+			func(config *datamodel.NodeBootstrappingConfiguration) {
+				config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = to.BoolPtr(true)
+				config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{
+					Name:                         "akscustom",
+					McrURL:                       "mcr.microsoft.fakecustomcloud",
+					RepoDepotEndpoint:            "https://repodepot.azure.microsoft.fakecustomcloud/ubuntu",
+					ManagementPortalURL:          "https://portal.azure.microsoft.fakecustomcloud/",
+					PublishSettingsURL:           "",
+					ServiceManagementEndpoint:    "https://management.core.microsoft.fakecustomcloud/",
+					ResourceManagerEndpoint:      "https://management.azure.microsoft.fakecustomcloud/",
+					ActiveDirectoryEndpoint:      "https://login.microsoftonline.microsoft.fakecustomcloud/",
+					GalleryEndpoint:              "",
+					KeyVaultEndpoint:             "https://vault.cloudapi.microsoft.fakecustomcloud/",
+					GraphEndpoint:                "https://graph.cloudapi.microsoft.fakecustomcloud/",
+					ServiceBusEndpoint:           "",
+					BatchManagementEndpoint:      "",
+					StorageEndpointSuffix:        "core.microsoft.fakecustomcloud",
+					SQLDatabaseDNSSuffix:         "database.cloudapi.microsoft.fakecustomcloud",
+					TrafficManagerDNSSuffix:      "",
+					KeyVaultDNSSuffix:            "vault.cloudapi.microsoft.fakecustomcloud",
+					ServiceBusEndpointSuffix:     "",
+					ServiceManagementVMDNSSuffix: "",
+					ResourceManagerVMDNSSuffix:   "cloudapp.azure.microsoft.fakecustomcloud/",
+					ContainerRegistryDNSSuffix:   ".azurecr.microsoft.fakecustomcloud",
+					CosmosDBDNSSuffix:            "documents.core.microsoft.fakecustomcloud/",
+					TokenAudience:                "https://management.core.microsoft.fakecustomcloud/",
+					ResourceIdentifiers: datamodel.ResourceIdentifiers{
+						Graph:               "",
+						KeyVault:            "",
+						Datalake:            "",
+						Batch:               "",
+						OperationalInsights: "",
+						Storage:             "",
+					},
+				}
+				config.KubeletConfig["--image-credential-provider-config"] = "c:\\var\\lib\\kubelet\\credential-provider-config.yaml"
+				config.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider"
+			}),
 	)
 
 })

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
+    H4sIAAAAAAAA/8STX2/aMBTF3/kUUcbbFruhq/jzFsBT0VioUjJpmqbIid3WWrCR48BWlO8+2RskMJNOa6W+Hh/73nPvz7uO4ziOmwmuMONUEnfk/NaMfsdyGuIVdUeOuxLpT682Jt3dZBEug1mIomnyGUW3s0VYvcWPpaTe0dlNsJxcHxyA0NR9V5cgYstzgclcZFgxwXUpKNYK1qXg3lPYLsbRXN95UGpdjCDUbYI0FynIhKRgyzgR2wJwqswRPIkBW2LA7i4ex+EyTiaLKQqDT6iCOePlD539Jk6CaHJdnT7YOpcyLbkqD89GaI6CW1SVLfNq1jqd3YbKggleuCPn67eGvmac0+NVGt0fXLzXs/JBH/ie7x5Oq8ZlSu7pwXS1d/1xuLLk2ROMaEvS3UVxODmX35z9e86zjOhSL0+HedUaAKZMcJado+Dlk///hn3g9+wLZrxQOM9t1wm9w2WurC8cYcA3jDBcY6+jK2YgOIPGU/tsWd7ZgdhaIpJpzys28r1MqeRU0ZYmao/HBaGeAcrDMnsACktw//iMLves46zwVkxKIYFhUH9sQ3pdHG66OwMjmiYf4zFKxrMwiL4c0IQp41gyWkB7w01y/+67OaITTEGvD/oNb60Orarfs8gDcGlVr6zqwKIOwYVVtVUbgsv6NxxvPFF0tc6xehX8O1XnDVp86PwKAAD//0Sfi7XLBwAA
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+Containerd/line77.sh
@@ -40,10 +40,13 @@
         "versions": [
             "1.27.7",
             "1.27.9",
+            "1.27.12",
             "1.28.3",
             "1.28.5",
+            "1.28.8",
             "1.29.0",
-            "1.29.2"
+            "1.29.2",
+            "1.29.3"
         ]
     },
     "_template": {

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
+    H4sIAAAAAAAA/8STX2/aMBTF3/kUUcbbFruhq/jzFsBT0VioUjJpmqbIid3WWrCR48BWlO8+2RskMJNOa6W+Hh/73nPvz7uO4ziOmwmuMONUEnfk/NaMfsdyGuIVdUeOuxLpT682Jt3dZBEug1mIomnyGUW3s0VYvcWPpaTe0dlNsJxcHxyA0NR9V5cgYstzgclcZFgxwXUpKNYK1qXg3lPYLsbRXN95UGpdjCDUbYI0FynIhKRgyzgR2wJwqswRPIkBW2LA7i4ex+EyTiaLKQqDT6iCOePlD539Jk6CaHJdnT7YOpcyLbkqD89GaI6CW1SVLfNq1jqd3YbKggleuCPn67eGvmac0+NVGt0fXLzXs/JBH/ie7x5Oq8ZlSu7pwXS1d/1xuLLk2ROMaEvS3UVxODmX35z9e86zjOhSL0+HedUaAKZMcJado+Dlk///hn3g9+wLZrxQOM9t1wm9w2WurC8cYcA3jDBcY6+jK2YgOIPGU/tsWd7ZgdhaIpJpzys28r1MqeRU0ZYmao/HBaGeAcrDMnsACktw//iMLves46zwVkxKIYFhUH9sQ3pdHG66OwMjmiYf4zFKxrMwiL4c0IQp41gyWkB7w01y/+67OaITTEGvD/oNb60Orarfs8gDcGlVr6zqwKIOwYVVtVUbgsv6NxxvPFF0tc6xehX8O1XnDVp86PwKAAD//0Sfi7XLBwAA
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line77.sh
@@ -40,10 +40,13 @@
         "versions": [
             "1.27.7",
             "1.27.9",
+            "1.27.12",
             "1.28.3",
             "1.28.5",
+            "1.28.8",
             "1.29.0",
-            "1.29.2"
+            "1.29.2",
+            "1.29.3"
         ]
     },
     "_template": {

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
+    H4sIAAAAAAAA/8STX2/aMBTF3/kUUcbbFruhq/jzFsBT0VioUjJpmqbIid3WWrCR48BWlO8+2RskMJNOa6W+Hh/73nPvz7uO4ziOmwmuMONUEnfk/NaMfsdyGuIVdUeOuxLpT682Jt3dZBEug1mIomnyGUW3s0VYvcWPpaTe0dlNsJxcHxyA0NR9V5cgYstzgclcZFgxwXUpKNYK1qXg3lPYLsbRXN95UGpdjCDUbYI0FynIhKRgyzgR2wJwqswRPIkBW2LA7i4ex+EyTiaLKQqDT6iCOePlD539Jk6CaHJdnT7YOpcyLbkqD89GaI6CW1SVLfNq1jqd3YbKggleuCPn67eGvmac0+NVGt0fXLzXs/JBH/ie7x5Oq8ZlSu7pwXS1d/1xuLLk2ROMaEvS3UVxODmX35z9e86zjOhSL0+HedUaAKZMcJado+Dlk///hn3g9+wLZrxQOM9t1wm9w2WurC8cYcA3jDBcY6+jK2YgOIPGU/tsWd7ZgdhaIpJpzys28r1MqeRU0ZYmao/HBaGeAcrDMnsACktw//iMLves46zwVkxKIYFhUH9sQ3pdHG66OwMjmiYf4zFKxrMwiL4c0IQp41gyWkB7w01y/+67OaITTEGvD/oNb60Orarfs8gDcGlVr6zqwKIOwYVVtVUbgsv6NxxvPFF0tc6xehX8O1XnDVp86PwKAAD//0Sfi7XLBwAA
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line77.sh
@@ -40,10 +40,13 @@
         "versions": [
             "1.27.7",
             "1.27.9",
+            "1.27.12",
             "1.28.3",
             "1.28.5",
+            "1.28.8",
             "1.29.0",
-            "1.29.2"
+            "1.29.2",
+            "1.29.3"
         ]
     },
     "_template": {

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
+    H4sIAAAAAAAA/8STX2/aMBTF3/kUUcbbFruhq/jzFsBT0VioUjJpmqbIid3WWrCR48BWlO8+2RskMJNOa6W+Hh/73nPvz7uO4ziOmwmuMONUEnfk/NaMfsdyGuIVdUeOuxLpT682Jt3dZBEug1mIomnyGUW3s0VYvcWPpaTe0dlNsJxcHxyA0NR9V5cgYstzgclcZFgxwXUpKNYK1qXg3lPYLsbRXN95UGpdjCDUbYI0FynIhKRgyzgR2wJwqswRPIkBW2LA7i4ex+EyTiaLKQqDT6iCOePlD539Jk6CaHJdnT7YOpcyLbkqD89GaI6CW1SVLfNq1jqd3YbKggleuCPn67eGvmac0+NVGt0fXLzXs/JBH/ie7x5Oq8ZlSu7pwXS1d/1xuLLk2ROMaEvS3UVxODmX35z9e86zjOhSL0+HedUaAKZMcJado+Dlk///hn3g9+wLZrxQOM9t1wm9w2WurC8cYcA3jDBcY6+jK2YgOIPGU/tsWd7ZgdhaIpJpzys28r1MqeRU0ZYmao/HBaGeAcrDMnsACktw//iMLves46zwVkxKIYFhUH9sQ3pdHG66OwMjmiYf4zFKxrMwiL4c0IQp41gyWkB7w01y/+67OaITTEGvD/oNb60Orarfs8gDcGlVr6zqwKIOwYVVtVUbgsv6NxxvPFF0tc6xehX8O1XnDVp86PwKAAD//0Sfi7XLBwAA
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line77.sh
@@ -40,10 +40,13 @@
         "versions": [
             "1.27.7",
             "1.27.9",
+            "1.27.12",
             "1.28.3",
             "1.28.5",
+            "1.28.8",
             "1.29.0",
-            "1.29.2"
+            "1.29.2",
+            "1.29.3"
         ]
     },
     "_template": {

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
+    H4sIAAAAAAAA/8STX2/aMBTF3/kUUcbbFruhq/jzFsBT0VioUjJpmqbIid3WWrCR48BWlO8+2RskMJNOa6W+Hh/73nPvz7uO4ziOmwmuMONUEnfk/NaMfsdyGuIVdUeOuxLpT682Jt3dZBEug1mIomnyGUW3s0VYvcWPpaTe0dlNsJxcHxyA0NR9V5cgYstzgclcZFgxwXUpKNYK1qXg3lPYLsbRXN95UGpdjCDUbYI0FynIhKRgyzgR2wJwqswRPIkBW2LA7i4ex+EyTiaLKQqDT6iCOePlD539Jk6CaHJdnT7YOpcyLbkqD89GaI6CW1SVLfNq1jqd3YbKggleuCPn67eGvmac0+NVGt0fXLzXs/JBH/ie7x5Oq8ZlSu7pwXS1d/1xuLLk2ROMaEvS3UVxODmX35z9e86zjOhSL0+HedUaAKZMcJado+Dlk///hn3g9+wLZrxQOM9t1wm9w2WurC8cYcA3jDBcY6+jK2YgOIPGU/tsWd7ZgdhaIpJpzys28r1MqeRU0ZYmao/HBaGeAcrDMnsACktw//iMLves46zwVkxKIYFhUH9sQ3pdHG66OwMjmiYf4zFKxrMwiL4c0IQp41gyWkB7w01y/+67OaITTEGvD/oNb60Orarfs8gDcGlVr6zqwKIOwYVVtVUbgsv6NxxvPFF0tc6xehX8O1XnDVp86PwKAAD//0Sfi7XLBwAA
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line77.sh
@@ -40,10 +40,13 @@
         "versions": [
             "1.27.7",
             "1.27.9",
+            "1.27.12",
             "1.28.3",
             "1.28.5",
+            "1.28.8",
             "1.29.0",
-            "1.29.2"
+            "1.29.2",
+            "1.29.3"
         ]
     },
     "_template": {

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
+    H4sIAAAAAAAA/8STX2/aMBTF3/kUUcbbFruhq/jzFsBT0VioUjJpmqbIid3WWrCR48BWlO8+2RskMJNOa6W+Hh/73nPvz7uO4ziOmwmuMONUEnfk/NaMfsdyGuIVdUeOuxLpT682Jt3dZBEug1mIomnyGUW3s0VYvcWPpaTe0dlNsJxcHxyA0NR9V5cgYstzgclcZFgxwXUpKNYK1qXg3lPYLsbRXN95UGpdjCDUbYI0FynIhKRgyzgR2wJwqswRPIkBW2LA7i4ex+EyTiaLKQqDT6iCOePlD539Jk6CaHJdnT7YOpcyLbkqD89GaI6CW1SVLfNq1jqd3YbKggleuCPn67eGvmac0+NVGt0fXLzXs/JBH/ie7x5Oq8ZlSu7pwXS1d/1xuLLk2ROMaEvS3UVxODmX35z9e86zjOhSL0+HedUaAKZMcJado+Dlk///hn3g9+wLZrxQOM9t1wm9w2WurC8cYcA3jDBcY6+jK2YgOIPGU/tsWd7ZgdhaIpJpzys28r1MqeRU0ZYmao/HBaGeAcrDMnsACktw//iMLves46zwVkxKIYFhUH9sQ3pdHG66OwMjmiYf4zFKxrMwiL4c0IQp41gyWkB7w01y/+67OaITTEGvD/oNb60Orarfs8gDcGlVr6zqwKIOwYVVtVUbgsv6NxxvPFF0tc6xehX8O1XnDVp86PwKAAD//0Sfi7XLBwAA
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+Docker/line77.sh
@@ -40,10 +40,13 @@
         "versions": [
             "1.27.7",
             "1.27.9",
+            "1.27.12",
             "1.28.3",
             "1.28.5",
+            "1.28.8",
             "1.29.0",
-            "1.29.2"
+            "1.29.2",
+            "1.29.3"
         ]
     },
     "_template": {

diff --git a/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
+    H4sIAAAAAAAA/8STX2/aMBTF3/kUUcbbFruhq/jzFsBT0VioUjJpmqbIid3WWrCR48BWlO8+2RskMJNOa6W+Hh/73nPvz7uO4ziOmwmuMONUEnfk/NaMfsdyGuIVdUeOuxLpT682Jt3dZBEug1mIomnyGUW3s0VYvcWPpaTe0dlNsJxcHxyA0NR9V5cgYstzgclcZFgxwXUpKNYK1qXg3lPYLsbRXN95UGpdjCDUbYI0FynIhKRgyzgR2wJwqswRPIkBW2LA7i4ex+EyTiaLKQqDT6iCOePlD539Jk6CaHJdnT7YOpcyLbkqD89GaI6CW1SVLfNq1jqd3YbKggleuCPn67eGvmac0+NVGt0fXLzXs/JBH/ie7x5Oq8ZlSu7pwXS1d/1xuLLk2ROMaEvS3UVxODmX35z9e86zjOhSL0+HedUaAKZMcJado+Dlk///hn3g9+wLZrxQOM9t1wm9w2WurC8cYcA3jDBcY6+jK2YgOIPGU/tsWd7ZgdhaIpJpzys28r1MqeRU0ZYmao/HBaGeAcrDMnsACktw//iMLves46zwVkxKIYFhUH9sQ3pdHG66OwMjmiYf4zFKxrMwiL4c0IQp41gyWkB7w01y/+67OaITTEGvD/oNb60Orarfs8gDcGlVr6zqwKIOwYVVtVUbgsv6NxxvPFF0tc6xehX8O1XnDVp86PwKAAD//0Sfi7XLBwAA
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/line77.sh
@@ -40,10 +40,13 @@
         "versions": [
             "1.27.7",
             "1.27.9",
+            "1.27.12",
             "1.28.3",
             "1.28.5",
+            "1.28.8",
             "1.29.0",
-            "1.29.2"
+            "1.29.2",
+            "1.29.3"
         ]
     },
     "_template": {

diff --git a/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
+    H4sIAAAAAAAA/8STX2/aMBTF3/kUUcbbFruhq/jzFsBT0VioUjJpmqbIid3WWrCR48BWlO8+2RskMJNOa6W+Hh/73nPvz7uO4ziOmwmuMONUEnfk/NaMfsdyGuIVdUeOuxLpT682Jt3dZBEug1mIomnyGUW3s0VYvcWPpaTe0dlNsJxcHxyA0NR9V5cgYstzgclcZFgxwXUpKNYK1qXg3lPYLsbRXN95UGpdjCDUbYI0FynIhKRgyzgR2wJwqswRPIkBW2LA7i4ex+EyTiaLKQqDT6iCOePlD539Jk6CaHJdnT7YOpcyLbkqD89GaI6CW1SVLfNq1jqd3YbKggleuCPn67eGvmac0+NVGt0fXLzXs/JBH/ie7x5Oq8ZlSu7pwXS1d/1xuLLk2ROMaEvS3UVxODmX35z9e86zjOhSL0+HedUaAKZMcJado+Dlk///hn3g9+wLZrxQOM9t1wm9w2WurC8cYcA3jDBcY6+jK2YgOIPGU/tsWd7ZgdhaIpJpzys28r1MqeRU0ZYmao/HBaGeAcrDMnsACktw//iMLves46zwVkxKIYFhUH9sQ3pdHG66OwMjmiYf4zFKxrMwiL4c0IQp41gyWkB7w01y/+67OaITTEGvD/oNb60Orarfs8gDcGlVr6zqwKIOwYVVtVUbgsv6NxxvPFF0tc6xehX8O1XnDVp86PwKAAD//0Sfi7XLBwAA
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"