feat: update window cse to install credental provider

.github/workflows/validate-windows-ut.yml

@@ -15,4 +15,5 @@ jobs:
       - name: Perform a Pester test from the staging/cse/windows/*.tests.ps1 file
         shell: powershell
         run: |
-          Invoke-Pester staging/cse/windows/*.tests.ps1 -Passthru
+          Install-Module -Name powershell-yaml -Force
+          Invoke-Pester staging/cse/windows/*.tests.ps1 -Passthru

staging/cse/windows/configfunc.ps1

@@ -342,6 +342,92 @@ function Install-OpenSSH {
     Write-Log "OpenSSH installed and configured successfully"
 }
 
+function Config-CredentialProvider {
+    Param(
+        [Parameter(Mandatory=$true)][string]
+        $KubeDir,
+        [Parameter(Mandatory = $false)][string]
+        $CustomCloudContainerRegistryDNSSuffix
+    )
+
+    $CredentialProviderConfPATH = [Io.path]::Combine("$KubeDir", "credential-provider-config.yaml")
+
+    Write-Log "Configuring kubelet credential provider"
+    $azureConfigFile = [io.path]::Combine("$KubeDir", "azure.json")
+
+    $credentialProviderConfig = @"
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - $azureConfigFile
+"@
+
+    if (![string]::IsNullOrEmpty($CustomCloudContainerRegistryDNSSuffix)) {
+        $credentialProviderConfig = @"
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+      - "*$CustomCloudContainerRegistryDNSSuffix"
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - $azureConfigFile
+"@
+    }
+    $credentialProviderConfig | Out-File -encoding ASCII -filepath "$CredentialProviderConfPATH"
+}
+
+function Install-CredentialProvider {
+    Param(
+        [Parameter(Mandatory=$true)][string]
+        $KubeDir,
+        [Parameter(Mandatory = $false)][string]
+        $CustomCloudContainerRegistryDNSSuffix
+    )
+
+    try {
+        $KubeletConfigArgsStr=$global:KubeletConfigArgs -join " "
+        # Out of tree credential provider is turned on as a must after 1.30, and is optinal in 1.29, for cluster < 1.29, it's not enabled.
+        # And only when it's enabled, the credential provider flags are set.
+        if ($KubeletConfigArgsStr -Like "*image-credential-provider-config*" -And $KubeletConfigArgsStr -Like "*image-credential-provider-bin-dir*") {
+            Logs-To-Event -TaskName "AKS.WindowsCSE.Install-CredentialProvider" -TaskMessage "Start to install out of tree credential provider"
+
+            Write-Log "Create credential provider configuration file"
+            Config-CredentialProvider -KubeDir $KubeDir -CustomCloudContainerRegistryDNSSuffix $CustomCloudContainerRegistryDNSSuffix
+
+            $CredentialProviderBinDir = "c:\var\lib\kubelet\credential-provider"
+            Write-Log "Download credential provider binary from $global:CredentialProviderURL to $CredentialProviderBinDir"
+            $tempDir = New-TemporaryDirectory
+            $credentialproviderbinaryPackage = "$tempDir\credentialprovider.tar.gz"
+            DownloadFileOverHttp -Url $global:CredentialProviderURL -DestinationPath $credentialproviderbinaryPackage -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER
+            tar -xzf $credentialproviderbinaryPackage -C $tempDir
+            Create-Directory -FullPath $CredentialProviderBinDir
+            cp "$tempDir\azure-acr-credential-provider.exe" "$CredentialProviderBinDir\acr-credential-provider.exe"
+            # acr-credential-provider.exe cannot be found by kubelet through provider name before the fix https://github.com/kubernetes/kubernetes/pull/120291
+            # so we copy the exe file to acr-credential-provider to make all 1.29 release work.
+            cp "$CredentialProviderBinDir\acr-credential-provider.exe" "$CredentialProviderBinDir\acr-credential-provider"
+            del $tempDir -Recurse
+        }
+    } catch {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER -ErrorMessage "Error installing credential provider. Error: $_"
+    }
+}
+
 function New-CsiProxyService {
     Param(
         [Parameter(Mandatory = $true)][string]

staging/cse/windows/configfunc.tests.ps1

@@ -104,4 +104,39 @@ Describe 'Resize-OSDrive' {
             Assert-MockCalled -CommandName "Set-ExitCode" -Exactly -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE }
         }
     }
+}
+
+Describe 'Config-CredentialProvider' {
+    BeforeEach {
+        $global:credentialProviderConfigDir = "staging\cse\windows\credentialProvider.tests.suites"
+        $CredentialProviderConfPATH=[Io.path]::Combine("$global:credentialProviderConfigDir", "credential-provider-config.yaml")
+        function Read-Format-Yaml ([string]$YamlFile) {
+            $yaml = Get-Content $YamlFile | ConvertFrom-Yaml
+            $yaml = $yaml | ConvertTo-Yaml
+            return $yaml
+        }
+    }
+
+    AfterEach {
+        Remove-Item -Path $CredentialProviderConfPATH
+    }
+
+    Context 'CustomCloudContainerRegistryDNSSuffix is empty' {
+        It "should match the expected config file content" {
+            $expectedCredentialProviderConfig = Read-Format-Yaml ([Io.path]::Combine($credentialProviderConfigDir, "CustomCloudContainerRegistryDNSSuffixEmpty.config.yaml"))
+            Config-CredentialProvider -KubeDir $credentialProviderConfigDir -CustomCloudContainerRegistryDNSSuffix ""
+            $acutalCredentialProviderConfig = Read-Format-Yaml $CredentialProviderConfPATH
+            $diffence = Compare-Object $acutalCredentialProviderConfig $expectedCredentialProviderConfig
+            $diffence | Should -Be $null
+        }
+    }
+   Context 'CustomCloudContainerRegistryDNSSuffix is not empty' {
+       It "should match the expected config file content" {
+            $expectedCredentialProviderConfig = Read-Format-Yaml ([Io.path]::Combine($credentialProviderConfigDir, "CustomCloudContainerRegistryDNSSuffixNotEmpty.config.yaml"))
+            Config-CredentialProvider -KubeDir $credentialProviderConfigDir -CustomCloudContainerRegistryDNSSuffix ".azurecr.microsoft.fakecloud"
+            $acutalCredentialProviderConfig = Read-Format-Yaml $CredentialProviderConfPATH
+            $diffence = Compare-Object $acutalCredentialProviderConfig $expectedCredentialProviderConfig
+            $diffence | Should -Be $null
+       }
+    }
 }

staging/cse/windows/credentialProvider.tests.suites/CustomCloudContainerRegistryDNSSuffixEmpty.config.yaml

@@ -0,0 +1,13 @@
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - staging\cse\windows\credentialProvider.tests.suites\azure.json

staging/cse/windows/credentialProvider.tests.suites/CustomCloudContainerRegistryDNSSuffixNotEmpty.config.yaml

@@ -0,0 +1,14 @@
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+      - "*.azurecr.microsoft.fakecloud"
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - staging\cse\windows\credentialProvider.tests.suites\azure.json

staging/cse/windows/debug/collect-windows-logs.ps1

@@ -77,6 +77,12 @@ if (Test-Path "c:\k\bootstrap-config") {
   $paths += "c:\k\bootstrap-config"
 }
 
+
+if (Test-Path "c:\k\credential-provider-config.yaml") {
+  Write-Host "Collecting credential provider config"
+  $paths += "c:\k\credential-provider-config.yaml"
+}
+
 Write-Host "Collecting Azure CNI configurations"
 $paths += "C:\k\azurecni\netconf\10-azure.conflist"
 $azureCNIConfigurations = @(

staging/cse/windows/provisioningscripts/loggenerator.ps1

@@ -81,7 +81,8 @@ $kLogFiles = @(
     "containerd.log",
     "containerd.err.log",
     "hnsremediator.log",
-    "windowslogscleanup.log"
+    "windowslogscleanup.log",
+    "credential-provider-config.yaml"
 )
 $kLogFiles | Foreach-Object {
     Create-SymbolLinkFile -SrcFile (Join-Path "C:\k\" $_) -DestFile (Join-Path $aksLogFolder $_)