feat: cse support credential provider and vhd cache binary for linux

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -66,6 +66,7 @@ MIG_NODE={{GetVariable "migNode"}}
 CONFIG_GPU_DRIVER_IF_NEEDED={{GetVariable "configGPUDriverIfNeeded"}}
 ENABLE_GPU_DEVICE_PLUGIN_IF_NEEDED={{GetVariable "enableGPUDevicePluginIfNeeded"}}
 TELEPORTD_PLUGIN_DOWNLOAD_URL={{GetParameter "teleportdPluginURL"}}
+CREDENTIAL_PROVIDER_DOWNLOAD_URL={{GetParameter "linuxCredentialProviderURL"}}
 CONTAINERD_VERSION={{GetParameter "containerdVersion"}}
 CONTAINERD_PACKAGE_URL={{GetParameter "containerdPackageURL"}}
 RUNC_VERSION={{GetParameter "runcVersion"}}
@@ -102,6 +103,7 @@ TARGET_CLOUD="{{- if IsAKSCustomCloud -}} AzureStackCloud {{- else -}} {{GetTarg
 TARGET_ENVIRONMENT="{{GetTargetEnvironment}}"
 CUSTOM_ENV_JSON="{{GetBase64EncodedEnvironmentJSON}}"
 IS_CUSTOM_CLOUD="{{IsAKSCustomCloud}}"
+AKSCustomCloudContainerRegistryDNSSuffix="{{- if IsAKSCustomCloud}}{{AKSCustomCloudContainerRegistryDNSSuffix}}{{end}}"
 CSE_HELPERS_FILEPATH="{{GetCSEHelpersScriptFilepath}}"
 CSE_DISTRO_HELPERS_FILEPATH="{{GetCSEHelpersScriptDistroFilepath}}"
 CSE_INSTALL_FILEPATH="{{GetCSEInstallScriptFilepath}}"

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -503,6 +503,14 @@ EOF
 # for DNS.
 iptables -I FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP
 EOF
+
+    # check if kubelet flags contain image-credential-provider-config and image-credential-provider-bin-dir
+    if [[ $KUBELET_FLAGS == *"image-credential-provider-config"* && $KUBELET_FLAGS == *"image-credential-provider-bin-dir"* ]]; then
+        echo "Configure credential provider for both image-credential-provider-config and image-credential-provider-bin-dir flags are specified in KUBELET_FLAGS"
+        logs_to_events "AKS.CSE.ensureKubelet.configCredentialProvider" configCredentialProvider
+        logs_to_events "AKS.CSE.ensureKubelet.installCredentalProvider" installCredentalProvider
+    fi
+
     systemctlEnableAndStart kubelet || exit $ERR_KUBELET_START_FAIL
 }
 
@@ -696,4 +704,32 @@ disableSSH() {
     systemctlDisableAndStop ssh || exit $ERR_DISABLE_SSH
 }
 
+configCredentialProvider() {
+    CREDENTIAL_PROVIDER_CONFIG_FILE=/var/lib/kubelet/credential-provider-config.yaml
+    mkdir -p "$(dirname "${CREDENTIAL_PROVIDER_CONFIG_FILE}")"
+    touch "${CREDENTIAL_PROVIDER_CONFIG_FILE}"
+    tee "${CREDENTIAL_PROVIDER_CONFIG_FILE}" > /dev/null <<EOF
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+$(if [[ -n "$AKSCustomCloudContainerRegistryDNSSuffix" ]]; then
+      echo "      - \"*$AKSCustomCloudContainerRegistryDNSSuffix\""
+fi)
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - /etc/kubernetes/azure.json
+EOF
+
+# remove blank lines from the config file.
+sed -i '/^$/d' "${CREDENTIAL_PROVIDER_CONFIG_FILE}"
+chmod 0600 "${CREDENTIAL_PROVIDER_CONFIG_FILE}"
+}
+
 #EOF

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -63,7 +63,7 @@ ERR_SYSTEMD_DOCKER_STOP_FAIL=116 # Error stopping dockerd
 ERR_CRICTL_DOWNLOAD_TIMEOUT=117 # Timeout waiting for crictl downloads
 ERR_CRICTL_OPERATION_ERROR=118 # Error executing a crictl operation
 ERR_CTR_OPERATION_ERROR=119 # Error executing a ctr containerd cli operation
-
+ERR_CREDENTIAL_PROVIDER_DOWNLOAD_TIMEOUT=120 # Timeout waiting for credential provider downloads
 # Azure Stack specific errors
 ERR_AZURE_STACK_GET_ARM_TOKEN=120 # Error generating a token to use with Azure Resource Manager
 ERR_AZURE_STACK_GET_NETWORK_CONFIGURATION=121 # Error fetching the network configuration for the node

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -15,6 +15,8 @@ UBUNTU_RELEASE=$(lsb_release -r -s)
 SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_DOWNLOAD_DIR="/opt/azure/tlsbootstrap"
 SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_VERSION="v0.1.0-alpha.2"
 TELEPORTD_PLUGIN_DOWNLOAD_DIR="/opt/teleportd/downloads"
+CREDENTIAL_PROVIDER_DOWNLOAD_DIR="/opt/credentialprovider/downloads"
+CREDENTIAL_PROVIDER_BIN_DIR="/var/lib/kubelet/credential-provider"
 TELEPORTD_PLUGIN_BIN_DIR="/usr/local/bin"
 CONTAINERD_WASM_VERSIONS="v0.3.0 v0.5.1 v0.8.0"
 MANIFEST_FILEPATH="/opt/azure/manifest.json"
@@ -74,6 +76,22 @@ downloadCNI() {
     retrycmd_get_tarball 120 5 "$CNI_DOWNLOADS_DIR/${CNI_TGZ_TMP}" ${CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT
 }
 
+downloadCredentalProvider() {
+    mkdir -p $CREDENTIAL_PROVIDER_DOWNLOAD_DIR
+    CREDENTIAL_PROVIDER_TGZ_TMP=${CREDENTIAL_PROVIDER_DOWNLOAD_URL##*/} # Use bash builtin ## to remove all chars ("*") up to the final "/"
+    retrycmd_get_tarball 120 5 "$CREDENTIAL_PROVIDER_DOWNLOAD_DIR/$CREDENTIAL_PROVIDER_TGZ_TMP" "$CREDENTIAL_PROVIDER_DOWNLOAD_URL" || exit $ERR_CREDENTIAL_PROVIDER_DOWNLOAD_TIMEOUT
+}
+
+installCredentalProvider() {
+    logs_to_events "AKS.CSE.installCredentalProvider.downloadCredentalProvider" downloadCredentalProvider
+    tar -xzf "$CREDENTIAL_PROVIDER_DOWNLOAD_DIR/${CREDENTIAL_PROVIDER_TGZ_TMP}" -C $CREDENTIAL_PROVIDER_DOWNLOAD_DIR
+    mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
+    chown -R root:root "${CREDENTIAL_PROVIDER_BIN_DIR}"
+    mv "${CREDENTIAL_PROVIDER_DOWNLOAD_DIR}/azure-acr-credential-provider" "${CREDENTIAL_PROVIDER_BIN_DIR}/acr-credential-provider"
+    chmod 755 "${CREDENTIAL_PROVIDER_BIN_DIR}/acr-credential-provider"
+    rm -rf ${CREDENTIAL_PROVIDER_DOWNLOAD_DIR}
+}
+
 downloadSecureTLSBootstrapKubeletExecPlugin() {
     local plugin_url="https://k8sreleases.blob.core.windows.net/aks-tls-bootstrap-client/${SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_VERSION}/linux/amd64/tls-bootstrap-client"
     if [[ $(isARM64) == 1 ]]; then

pkg/agent/baker_test.go

@@ -123,6 +123,11 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 			WindowsPackageURL:         windowsPackage,
 		}
 
+		if IsKubernetesVersionGe(k8sVersion, "1.29.0") {
+			k8sComponents.WindowsCredentialProviderURL = fmt.Sprintf("https://acs-mirror.azureedge.net/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s", k8sVersion, k8sVersion) //nolint:lll
+			k8sComponents.LinuxCredentialProviderURL = fmt.Sprintf("https://acs-mirror.azureedge.net/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-windows-amd64-v%s", k8sVersion, k8sVersion) //nolint:lll
+		}
+
 		kubeletConfig := map[string]string{
 			"--address":                           "0.0.0.0",
 			"--pod-manifest-path":                 "/etc/kubernetes/manifests",
@@ -1214,6 +1219,60 @@ oom_score = 0
 				Name: "akscustom",
 			}
 		}, nil),
+		Entry("AKSUbuntu2204 OOT credentialprovider", "AKSUbuntu2204+ootcredentialprovider", "1.29.10", func(config *datamodel.NodeBootstrappingConfiguration) {
+			config.KubeletConfig["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml"
+			config.KubeletConfig["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider"
+		}, func(o *nodeBootstrappingOutput) {
+			Expect(o.vars["KUBELET_FLAGS"]).NotTo(BeEmpty())
+			Expect(strings.Contains(o.vars["KUBELET_FLAGS"], "--image-credential-provider-config=/var/lib/kubelet/credential-provider-config.yaml")).To(BeTrue())
+			Expect(strings.Contains(o.vars["KUBELET_FLAGS"], "--image-credential-provider-bin-dir=/var/lib/kubelet/credential-provider")).To(BeTrue())
+		}),
+		Entry("AKSUbuntu2204 custom cloud and OOT credentialprovider", "AKSUbuntu2204+CustomCloud+ootcredentialprovider", "1.29.10",
+			func(config *datamodel.NodeBootstrappingConfiguration) {
+				config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{
+					Name:                         "akscustom",
+					McrURL:                       "mcr.microsoft.fakecustomcloud",
+					RepoDepotEndpoint:            "https://repodepot.azure.microsoft.fakecustomcloud/ubuntu",
+					ManagementPortalURL:          "https://portal.azure.microsoft.fakecustomcloud/",
+					PublishSettingsURL:           "",
+					ServiceManagementEndpoint:    "https://management.core.microsoft.fakecustomcloud/",
+					ResourceManagerEndpoint:      "https://management.azure.microsoft.fakecustomcloud/",
+					ActiveDirectoryEndpoint:      "https://login.microsoftonline.microsoft.fakecustomcloud/",
+					GalleryEndpoint:              "",
+					KeyVaultEndpoint:             "https://vault.cloudapi.microsoft.fakecustomcloud/",
+					GraphEndpoint:                "https://graph.cloudapi.microsoft.fakecustomcloud/",
+					ServiceBusEndpoint:           "",
+					BatchManagementEndpoint:      "",
+					StorageEndpointSuffix:        "core.microsoft.fakecustomcloud",
+					SQLDatabaseDNSSuffix:         "database.cloudapi.microsoft.fakecustomcloud",
+					TrafficManagerDNSSuffix:      "",
+					KeyVaultDNSSuffix:            "vault.cloudapi.microsoft.fakecustomcloud",
+					ServiceBusEndpointSuffix:     "",
+					ServiceManagementVMDNSSuffix: "",
+					ResourceManagerVMDNSSuffix:   "cloudapp.azure.microsoft.fakecustomcloud/",
+					ContainerRegistryDNSSuffix:   ".azurecr.microsoft.fakecustomcloud",
+					CosmosDBDNSSuffix:            "documents.core.microsoft.fakecustomcloud/",
+					TokenAudience:                "https://management.core.microsoft.fakecustomcloud/",
+					ResourceIdentifiers: datamodel.ResourceIdentifiers{
+						Graph:               "",
+						KeyVault:            "",
+						Datalake:            "",
+						Batch:               "",
+						OperationalInsights: "",
+						Storage:             "",
+					},
+				}
+				config.KubeletConfig["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml"
+				config.KubeletConfig["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider"
+			}, func(o *nodeBootstrappingOutput) {
+
+				Expect(o.vars["AKSCustomCloudContainerRegistryDNSSuffix"]).NotTo(BeEmpty())
+				Expect(o.vars["AKSCustomCloudContainerRegistryDNSSuffix"]).To(Equal(".azurecr.microsoft.fakecustomcloud"))
+
+				Expect(o.vars["KUBELET_FLAGS"]).NotTo(BeEmpty())
+				Expect(strings.Contains(o.vars["KUBELET_FLAGS"], "--image-credential-provider-config=/var/lib/kubelet/credential-provider-config.yaml")).To(BeTrue())
+				Expect(strings.Contains(o.vars["KUBELET_FLAGS"], "--image-credential-provider-bin-dir=/var/lib/kubelet/credential-provider")).To(BeTrue())
+			}),
 		Entry("AKSUbuntu2204 with custom kubeletConfig and osConfig", "AKSUbuntu2204+CustomKubeletConfig+CustomLinuxOSConfig", "1.24.2",
 			func(config *datamodel.NodeBootstrappingConfiguration) {
 				config.EnableKubeletConfigFile = false
@@ -1371,6 +1430,12 @@ var _ = Describe("Assert generated customData and cseCmd for Windows", func() {
 			WindowsPackageURL:         windowsPackage,
 		}
 
+		if IsKubernetesVersionGe(k8sVersion, "1.29.0") {
+			// This is test only, credential provider version does not align with k8s version
+			k8sComponents.WindowsCredentialProviderURL = fmt.Sprintf("https://acs-mirror.azureedge.net/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s", k8sVersion, k8sVersion) //nolint:lll
+			k8sComponents.LinuxCredentialProviderURL = fmt.Sprintf("https://acs-mirror.azureedge.net/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-windows-amd64-v%s", k8sVersion, k8sVersion) //nolint:lll
+		}
+
 		kubeletConfig := map[string]string{
 			"--address":                           "0.0.0.0",
 			"--anonymous-auth":                    "false",
@@ -1551,6 +1616,50 @@ var _ = Describe("Assert generated customData and cseCmd for Windows", func() {
 				},
 			}
 		}),
+		Entry("AKSWindows2019 with out of tree credential provider", "AKSWindows2019+ootcredentialprovider", "1.19.0", func(config *datamodel.NodeBootstrappingConfiguration) {
+			config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = to.BoolPtr(true)
+			config.KubeletConfig["--image-credential-provider-config"] = "c:\\var\\lib\\kubelet\\credential-provider-config.yaml"
+			config.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider"
+		}),
+		Entry("AKSWindows2019 with custom cloud and out of tree credential provider", "AKSWindows2019+CustomCloud+ootcredentialprovider", "1.19.0",
+			func(config *datamodel.NodeBootstrappingConfiguration) {
+				config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = to.BoolPtr(true)
+				config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{
+					Name:                         "akscustom",
+					McrURL:                       "mcr.microsoft.fakecustomcloud",
+					RepoDepotEndpoint:            "https://repodepot.azure.microsoft.fakecustomcloud/ubuntu",
+					ManagementPortalURL:          "https://portal.azure.microsoft.fakecustomcloud/",
+					PublishSettingsURL:           "",
+					ServiceManagementEndpoint:    "https://management.core.microsoft.fakecustomcloud/",
+					ResourceManagerEndpoint:      "https://management.azure.microsoft.fakecustomcloud/",
+					ActiveDirectoryEndpoint:      "https://login.microsoftonline.microsoft.fakecustomcloud/",
+					GalleryEndpoint:              "",
+					KeyVaultEndpoint:             "https://vault.cloudapi.microsoft.fakecustomcloud/",
+					GraphEndpoint:                "https://graph.cloudapi.microsoft.fakecustomcloud/",
+					ServiceBusEndpoint:           "",
+					BatchManagementEndpoint:      "",
+					StorageEndpointSuffix:        "core.microsoft.fakecustomcloud",
+					SQLDatabaseDNSSuffix:         "database.cloudapi.microsoft.fakecustomcloud",
+					TrafficManagerDNSSuffix:      "",
+					KeyVaultDNSSuffix:            "vault.cloudapi.microsoft.fakecustomcloud",
+					ServiceBusEndpointSuffix:     "",
+					ServiceManagementVMDNSSuffix: "",
+					ResourceManagerVMDNSSuffix:   "cloudapp.azure.microsoft.fakecustomcloud/",
+					ContainerRegistryDNSSuffix:   ".azurecr.microsoft.fakecustomcloud",
+					CosmosDBDNSSuffix:            "documents.core.microsoft.fakecustomcloud/",
+					TokenAudience:                "https://management.core.microsoft.fakecustomcloud/",
+					ResourceIdentifiers: datamodel.ResourceIdentifiers{
+						Graph:               "",
+						KeyVault:            "",
+						Datalake:            "",
+						Batch:               "",
+						OperationalInsights: "",
+						Storage:             "",
+					},
+				}
+				config.KubeletConfig["--image-credential-provider-config"] = "c:\\var\\lib\\kubelet\\credential-provider-config.yaml"
+				config.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider"
+			}),
 		Entry("AKSWindows2019 EnablePrivateClusterHostsConfigAgent", "AKSWindows2019+EnablePrivateClusterHostsConfigAgent", "1.19.0",
 			func(config *datamodel.NodeBootstrappingConfiguration) {
 				cs := config.ContainerService

pkg/agent/datamodel/azenvtypes.go

@@ -42,6 +42,8 @@ type KubernetesSpecConfig struct {
 	CseScriptsPackageURL                 string `json:"cseScriptsPackageURL,omitempty"`
 	CNIARM64PluginsDownloadURL           string `json:"cniARM64PluginsDownloadURL,omitempty"`
 	VnetCNIARM64LinuxPluginsDownloadURL  string `json:"vnetCNIARM64LinuxPluginsDownloadURL,omitempty"`
+	CredentialProviderURL                string `json:"credentialProviderURL,omitempty"`
+	CredentialProviderWindowsURL         string `json:"credentialProviderWindowsURL,omitempty"`
 }
 
 // AzureEndpointConfig describes an Azure endpoint.

pkg/agent/datamodel/types.go

@@ -1620,6 +1620,14 @@ type K8sComponents struct {
 	// Full path to the Linux package (tar.gz) to use.
 	// For example: url=https://acs-mirror.azureedge.net/kubernetes/v1.25.6-hotfix.20230612/binaries/v1.25.6-hotfix.20230612.tar.gz
 	LinuxPrivatePackageURL string
+
+	// Full path to the Windows credential provider (tar.gz) to use.
+	// For example: https://acs-mirror.azureedge.net/cloud-provider-azure/v1.29.4/binaries/azure-acr-credential-provider-linux-amd64-v1.29.4.tar.gz
+	WindowsCredentialProviderURL string
+
+	// Full path to the Linux credential provider (tar.gz) to use.
+	// For example: "https://acs-mirror.azureedge.net/cloud-provider-azure/v1.29.4/binaries/azure-acr-credential-provider-windows-amd64-v1.29.4.tar.gz"
+	LinuxCredentialProviderURL string
 }
 
 // GetLatestSigImageConfigRequest describes the input for a GetLatestSigImageConfig HTTP request.

pkg/agent/params.go

@@ -165,15 +165,17 @@ func assignKubernetesParametersfromKubernetesConfig(properties *datamodel.Proper
 		addValue(parametersMap, "vnetCniLinuxPluginsURL", kubernetesConfig.GetAzureCNIURLLinux(cloudSpecConfig))
 	}
 	addValue(parametersMap, "vnetCniWindowsPluginsURL", kubernetesConfig.GetAzureCNIURLWindows(cloudSpecConfig))
+	addValue(parametersMap, "linuxCredentialProviderURL", k8sComponents.LinuxCredentialProviderURL)
 
 	if properties.HasWindows() {
 		addValue(parametersMap, "kubeBinariesSASURL", k8sComponents.WindowsPackageURL)
-
+		addValue(parametersMap, "credentialProviderWindowsURL", cloudSpecConfig.KubernetesSpecConfig.CredentialProviderWindowsURL)
 		addValue(parametersMap, "windowsContainerdURL", kubernetesConfig.WindowsContainerdURL)
 		addValue(parametersMap, "kubeServiceCidr", kubernetesConfig.ServiceCIDR)
 		addValue(parametersMap, "kubeBinariesVersion", k8sVersion)
 		addValue(parametersMap, "windowsTelemetryGUID", cloudSpecConfig.KubernetesSpecConfig.WindowsTelemetryGUID)
 		addValue(parametersMap, "windowsSdnPluginURL", kubernetesConfig.WindowsSdnPluginURL)
+		addValue(parametersMap, "windowsCredentialProviderURL", k8sComponents.WindowsCredentialProviderURL)
 	}
 }