Merge branch 'master' of github.com:Azure/AgentBaker into qinhao/cred…

…ential-provider-windows

.github/workflows/validate-windows-ut.yml

@@ -15,4 +15,5 @@ jobs:
       - name: Perform a Pester test from the staging/cse/windows/*.tests.ps1 file
         shell: powershell
         run: |
-          Invoke-Pester staging/cse/windows/*.tests.ps1 -Passthru
+          Install-Module -Name powershell-yaml -Force
+          Invoke-Pester staging/cse/windows/*.tests.ps1 -Passthru

.pipelines/templates/.builder-release-template-windows.yaml

@@ -38,9 +38,9 @@ steps:
           echo "##vso[task.setvariable variable=SIG_FOR_PRODUCTION]False"; \
           echo "##vso[task.setvariable variable=SKIPVALIDATEREOFFERUPDATE]True"; \
       else
-          sigImageName="windows-${WINDOWS_SKU}-$RANDOM"; \
+          sigImageName="windows-${WINDOWS_SKU}-$(date +"%H%M%S")-$RANDOM"; \
           WS_SKU=$(echo $WINDOWS_SKU | tr '-' '_')
-          sigGalleryName="WS${WS_SKU}Gallery$(date +"%y%m%d%H%M%S")$RANDOM"; \
+          sigGalleryName="WSGallery$(date +"%y%m%d")";\
           echo "##vso[task.setvariable variable=SIG_GALLERY_NAME]$sigGalleryName"; \
           echo "##vso[task.setvariable variable=SIG_IMAGE_VERSION]1.0.0"; \
           echo "##vso[task.setvariable variable=SIG_FOR_PRODUCTION]True"; \

parts/windows/kuberneteswindowssetup.ps1

@@ -239,7 +239,7 @@ try
     Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'"
     # TODO update to use proxy
 
-    $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.40.zip"
+    $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.41.zip"
     Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl"
     Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage"
     # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl

parts/windows/windowscsehelper.ps1

@@ -68,6 +68,9 @@ $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61
 $global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62
 $global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63
 $global:WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION=64
+$global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER = 65 # exit code for installing credential provider
+$global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER=66 # exit code for downloading credential provider failure
+$global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG=67 # exit code for checking credential provider config failure
 
 # Please add new error code for downloading new packages in RP code too
 $global:ErrorCodeNames = @(
@@ -135,7 +138,10 @@ $global:ErrorCodeNames = @(
     "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE",
     "WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG",
     "WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP",
-    "WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION"
+    "WINDOWS_CSE_ERROR_GET_CONTAINERD_VERSION",
+    "WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER",
+    "WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG"
 )
 
 # NOTE: KubernetesVersion does not contain "v"

staging/cse/windows/README

@@ -55,6 +55,15 @@ popd
 # AKS Windows CSE Scripts Package
 All files except *.test.ps1 and README will be published in AKS Windows CSE Scripts Package.
 
+## v0.0.41
+- feat: update window cse to install credental provider #4281
+- Add more containerd logs to collect-windows-logs.ps1 #3952
+- feat: add system startup task to expand os volume #4213
+- feat: refine the code to get the Windows containerd version #4226
+- feat: collect Windows extensions logs #4169
+- feat: collect Windows memroy dump in collect-windows-logs.ps1 #4168
+- fix: avoid running Windows CSE multiple times. #4158
+
 ## v0.0.40
 - feat: set --node-ip for Windows kubelet in k8s v1.29+ #4148
 

staging/cse/windows/configfunc.ps1

@@ -342,6 +342,123 @@ function Install-OpenSSH {
     Write-Log "OpenSSH installed and configured successfully"
 }
 
+function Config-CredentialProvider {
+    Param(
+        [Parameter(Mandatory=$true)][string]
+        $KubeDir,
+        [Parameter(Mandatory=$true)][string]
+        $CredentialProviderConfPath,
+        [Parameter(Mandatory = $false)][string]
+        $CustomCloudContainerRegistryDNSSuffix
+    )
+
+    Write-Log "Configuring kubelet credential provider"
+    $azureConfigFile = [io.path]::Combine("$KubeDir", "azure.json")
+
+    $credentialProviderConfig = @"
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - $azureConfigFile
+"@
+
+    if (![string]::IsNullOrEmpty($CustomCloudContainerRegistryDNSSuffix)) {
+        $credentialProviderConfig = @"
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+      - "*$CustomCloudContainerRegistryDNSSuffix"
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - $azureConfigFile
+"@
+    }
+    $credentialProviderConfig | Out-File -encoding ASCII -filepath "$CredentialProviderConfPATH"
+}
+
+function Validate-CredentialProviderConfigFlags {
+    function get-KubeletFlagValue {
+        Param(
+            [Parameter(Mandatory=$true)][string]
+            $KubeletConfigArg
+        )
+        $splitResult=($KubeletConfigArg -split "=")
+        if ($splitResult.Length -ne 2 -or [string]::IsNullOrEmpty($splitResult[1])){
+            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG -ErrorMessage "Failed to get kubelet flag value from flag $KubeletConfigArg"
+        }
+        return $splitResult[1]
+    }
+    ForEach ($kubeletConfigArg in $global:KubeletConfigArgs){
+        if ($kubeletConfigArg -like "--image-credential-provider-config=*") {
+            $global:credentialProviderConfigPath=get-KubeletFlagValue -KubeletConfigArg $kubeletConfigArg
+        }
+        if ($kubeletConfigArg -like "--image-credential-provider-bin-dir=*") {
+            $global:credentialProviderBinDir=get-KubeletFlagValue -KubeletConfigArg $kubeletConfigArg
+        }
+    }
+
+    # Both flags should be set to enable out of tree credential provider or not set at the same time to disable it.
+    if ([string]::IsNullOrEmpty($credentialProviderConfigPath) -xor [string]::IsNullOrEmpty($credentialProviderBinDir)) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG -ErrorMessage "Not all credential provider flags are configured: --image-credential-provider-config=$credentialProviderConfigPath, --image-credential-provider-bin-dir=$credentialProviderBinDir"
+    }
+}
+
+function Install-CredentialProvider {
+    Param(
+        [Parameter(Mandatory=$true)][string]
+        $KubeDir,
+        [Parameter(Mandatory = $false)][string]
+        $CustomCloudContainerRegistryDNSSuffix
+    )
+
+    try {
+        # Out of tree credential provider is turned on as a must after 1.30, and is optinal in 1.29, for cluster < 1.29, it's not enabled.
+        # And only when it's enabled, the credential provider flags are set.
+        $global:credentialProviderConfigPath = ""
+        $global:credentialProviderBinDir = ""
+        Validate-CredentialProviderConfigFlags
+        if ([string]::IsNullOrEmpty($global:credentialProviderConfigPath) -and [string]::IsNullOrEmpty($global:credentialProviderBinDir)) {
+            Write-Log "Out of tree credential provider is not enabled"
+            return
+        }
+
+        Logs-To-Event -TaskName "AKS.WindowsCSE.Install-CredentialProvider" -TaskMessage "Start to install out of tree credential provider"
+
+        Write-Log "Create credential provider configuration file"
+        Config-CredentialProvider -KubeDir $KubeDir -CredentialProviderConfPath $global:credentialProviderConfigPath -CustomCloudContainerRegistryDNSSuffix $CustomCloudContainerRegistryDNSSuffix
+
+        Write-Log "Download credential provider binary from $global:CredentialProviderURL to $global:credentialProviderBinDir"
+        $tempDir = New-TemporaryDirectory
+        $credentialproviderbinaryPackage = "$tempDir\credentialprovider.tar.gz"
+        DownloadFileOverHttp -Url $global:CredentialProviderURL -DestinationPath $credentialproviderbinaryPackage -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER
+        tar -xzf $credentialproviderbinaryPackage -C $tempDir
+        Create-Directory -FullPath $global:credentialProviderBinDir
+        cp "$tempDir\azure-acr-credential-provider.exe" "$global:credentialProviderBinDir\acr-credential-provider.exe"
+        # acr-credential-provider.exe cannot be found by kubelet through provider name before the fix https://github.com/kubernetes/kubernetes/pull/120291
+        # so we copy the exe file to acr-credential-provider to make all 1.29 release work.
+        cp "$global:credentialProviderBinDir\acr-credential-provider.exe" "$global:credentialProviderBinDir\acr-credential-provider"
+        del $tempDir -Recurse
+    } catch {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER -ErrorMessage "Error installing credential provider. Error: $_"
+    }
+}
+
 function New-CsiProxyService {
     Param(
         [Parameter(Mandatory = $true)][string]

staging/cse/windows/configfunc.tests.ps1

@@ -104,4 +104,99 @@ Describe 'Resize-OSDrive' {
             Assert-MockCalled -CommandName "Set-ExitCode" -Exactly -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE }
         }
     }
+}
+
+Describe 'Config-CredentialProvider' {
+    BeforeEach {
+        $global:credentialProviderConfigDir = "staging\cse\windows\credentialProvider.tests.suites"
+        $CredentialProviderConfPATH=[Io.path]::Combine("$global:credentialProviderConfigDir", "credential-provider-config.yaml")
+        function Read-Format-Yaml ([string]$YamlFile) {
+            $yaml = Get-Content $YamlFile | ConvertFrom-Yaml
+            $yaml = $yaml | ConvertTo-Yaml
+            return $yaml
+        }
+    }
+
+    AfterEach {
+        Remove-Item -Path $CredentialProviderConfPATH
+    }
+
+    Context 'CustomCloudContainerRegistryDNSSuffix is empty' {
+        It "should match the expected config file content" {
+            $expectedCredentialProviderConfig = Read-Format-Yaml ([Io.path]::Combine($credentialProviderConfigDir, "CustomCloudContainerRegistryDNSSuffixEmpty.config.yaml"))
+            Config-CredentialProvider -KubeDir $credentialProviderConfigDir -CredentialProviderConfPath $CredentialProviderConfPATH -CustomCloudContainerRegistryDNSSuffix ""
+
+            $acutalCredentialProviderConfig = Read-Format-Yaml $CredentialProviderConfPATH
+            $diffence = Compare-Object $acutalCredentialProviderConfig $expectedCredentialProviderConfig
+            $diffence | Should -Be $null
+        }
+    }
+   Context 'CustomCloudContainerRegistryDNSSuffix is not empty' {
+       It "should match the expected config file content" {
+            $expectedCredentialProviderConfig = Read-Format-Yaml ([Io.path]::Combine($credentialProviderConfigDir, "CustomCloudContainerRegistryDNSSuffixNotEmpty.config.yaml"))
+            Config-CredentialProvider -KubeDir $credentialProviderConfigDir -CredentialProviderConfPath $CredentialProviderConfPATH -CustomCloudContainerRegistryDNSSuffix ".azurecr.microsoft.fakecloud"
+            $acutalCredentialProviderConfig = Read-Format-Yaml $CredentialProviderConfPATH
+            $diffence = Compare-Object $acutalCredentialProviderConfig $expectedCredentialProviderConfig
+            $diffence | Should -Be $null
+       }
+    }
+}
+
+Describe 'Validate-CredentialProviderConfigFlags' {
+    BeforeEach {
+        $global:KubeletConfigArgs = @( "--address=0.0.0.0" )
+        $global:credentialProviderConfigPath = ""
+        $global:credentialProviderBinDir = ""
+    }
+
+    BeforeAll{
+        Mock Set-ExitCode -MockWith {
+            Param(
+              $ExitCode,
+              $ErrorMessage
+            )
+            Write-Host "Set-ExitCode $ExitCode $ErrorMessage"
+        } -Verifiable
+    }
+
+    Context 'success' {
+        It "Should return expected config path and bin path" {
+            $expectedCredentialProviderConfigPath="c:\k\credential-provider-config.yaml"
+            $expectedCredentialProviderBinDir="c:\var\lib\kubelet\credential-provider"
+            $global:KubeletConfigArgs+="--image-credential-provider-config="+$expectedCredentialProviderConfigPath
+            $global:KubeletConfigArgs+="--image-credential-provider-bin-dir="+$expectedCredentialProviderBinDir
+            Validate-CredentialProviderConfigFlags
+            Compare-Object $global:credentialProviderConfigPath $expectedCredentialProviderConfigPath | Should -Be $null
+            Compare-Object $global:credentialProviderBinDir $expectedCredentialProviderBinDir | Should -Be $null
+        }
+
+        It "Should return empty config path and bin path" {
+            $expectedCredentialProviderConfigPath=""
+            $expectedCredentialProviderBinDir=""
+            Validate-CredentialProviderConfigFlags
+            Compare-Object $global:credentialProviderConfigPath $expectedCredentialProviderConfigPath | Should -Be $null
+            Compare-Object $global:credentialProviderBinDir $expectedCredentialProviderBinDir | Should -Be $null
+        }
+    }
+
+    Context 'fail' {
+        It "Should call Set-ExitCode when only config path is specified" {
+            $expectedCredentialProviderConfigPath="c:\k\credential-provider_config.yaml"
+            $global:KubeletConfigArgs+="--image-credential-provider-config="+$expectedCredentialProviderConfigPath
+            $credentialProviderConfigs = Validate-CredentialProviderConfigFlags
+            Assert-MockCalled -CommandName "Set-ExitCode" -Exactly -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG }
+        }
+        It "Should call Set-ExitCode when only bin dir is specified" {
+            $expectedCredentialProviderBinDir="c:\var\lib\kubelet\credential-provider"
+            $global:KubeletConfigArgs+="--image-credential-provider-bin-dir="+$expectedCredentialProviderBinDir
+            $credentialProviderConfigs = Validate-CredentialProviderConfigFlags
+            Assert-MockCalled -CommandName "Set-ExitCode" -Exactly -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG }
+        }
+        It "Should call Set-ExitCode when flag value is emtpy string" {
+            $expectedCredentialProviderBinDir="c:\var\lib\kubelet\credential-provider"
+            $global:KubeletConfigArgs+="--image-credential-provider-bin-dir="
+            $credentialProviderConfigs = Validate-CredentialProviderConfigFlags
+            Assert-MockCalled -CommandName "Set-ExitCode" -Exactly -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG }
+        }
+    }
 }

staging/cse/windows/credentialProvider.tests.suites/CustomCloudContainerRegistryDNSSuffixEmpty.config.yaml

@@ -0,0 +1,13 @@
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - staging\cse\windows\credentialProvider.tests.suites\azure.json

staging/cse/windows/credentialProvider.tests.suites/CustomCloudContainerRegistryDNSSuffixNotEmpty.config.yaml

@@ -0,0 +1,14 @@
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+      - "*.azurecr.microsoft.fakecloud"
+    defaultCacheDuration: "10m"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - staging\cse\windows\credentialProvider.tests.suites\azure.json

staging/cse/windows/debug/collect-windows-logs.ps1

@@ -77,6 +77,12 @@ if (Test-Path "c:\k\bootstrap-config") {
   $paths += "c:\k\bootstrap-config"
 }
 
+
+if (Test-Path "c:\k\credential-provider-config.yaml") {
+  Write-Host "Collecting credential provider config"
+  $paths += "c:\k\credential-provider-config.yaml"
+}
+
 Write-Host "Collecting Azure CNI configurations"
 $paths += "C:\k\azurecni\netconf\10-azure.conflist"
 $azureCNIConfigurations = @(

staging/cse/windows/debug/collectlogs.ps1

@@ -2,7 +2,8 @@
 
 Param(
     [parameter(Mandatory = $false)] [string] $Network = "L2Bridge",
-    [parameter(Mandatory = $false)] [ValidateSet(1,2)] [int] $HnsSchemaVersion = 2
+    [parameter(Mandatory = $false)] [ValidateSet(1,2)] [int] $HnsSchemaVersion = 2,
+    [parameter(Mandatory = $false)] [string] $outDir
 )
 
 $GithubSDNRepository = 'Microsoft/SDN'
@@ -22,9 +23,11 @@ if (!(Test-Path $helper))
 
 ipmo $BaseDir\hns.v2.psm1 -Force
 
-$ScriptPath = Split-Path $MyInvocation.MyCommand.Path
+if ([System.String]::IsNullOrEmpty($outDir)) {
+    $ScriptPath = Split-Path $MyInvocation.MyCommand.Path
+    $outDir = [io.Path]::Combine($ScriptPath, [io.Path]::GetRandomFileName())
+}
 
-$outDir = [io.Path]::Combine($ScriptPath, [io.Path]::GetRandomFileName())
 md $outDir
 pushd 
 cd $outDir

staging/cse/windows/debug/dumpVfpPolicies.ps1

@@ -29,9 +29,10 @@ foreach ($port in $ports) {
 	echo "Policy for port : " $portGuid  | Out-File $outfile -Encoding ascii -Append
 	& $vfpCtrlExe /list-space  /port $portGuid | Out-File $outfile -Encoding ascii -Append
 	& $vfpCtrlExe /list-mapping  /port $portGuid | Out-File $outfile -Encoding ascii -Append
-	& $vfpCtrlExe /list-rule  /port $portGuid | Out-File $outfile -Encoding ascii -Append
+	& $vfpCtrlExe /port $portGuid /get-rule-counter | Out-File $outfile -Encoding ascii -Append
 	& $vfpCtrlExe /port $portGuid /get-port-state | Out-File $outfile -Encoding ascii -Append
 	& $vfpCtrlExe /port $portGuid /list-nat-range | Out-File $outfile -Encoding ascii -Append
+	& $vfpCtrlExe /port $portGuid /get-flow-stats | Out-File $outfile -Encoding ascii -Append
 }
 
-& $vfpCtrlExe /switch $ports[0].'Switch Name'  /get-switch-forwarding-settings > vswitchForwarding.txt
+& $vfpCtrlExe /switch $ports[0].'Switch Name'  /get-switch-forwarding-settings > vswitchForwarding.txt