Merge branch 'alburgess/cse-security' of https://github.com/Azure/Age…

…ntBaker into alburgess/cse-security
Azure · Mar 14, 2024 · 4c7f9b5 · 4c7f9b5
2 parents 86674ad + 2238f97
commit 4c7f9b5
Show file tree

Hide file tree

Showing 19 changed files with 93 additions and 31 deletions.
diff --git a/.pipelines/e2e.yaml b/.pipelines/e2e.yaml
@@ -15,6 +15,7 @@ pr:
     - e2e/windows
     - pkg/agent/datamodel/sig_config*.go # SIG config changes
     - pkg/agent/datamodel/*.json # SIG version changes
+    - pkg/agent/testdata/AKSWindows* # Windows test data
 
 variables:
 - group: ab-e2e

diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1
@@ -232,7 +232,7 @@ try
     Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'"
     # TODO update to use proxy
 
-    $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.39.zip"
+    $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.40.zip"
     Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl"
     Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage"
     # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl

diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1
@@ -65,6 +65,8 @@ $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE=58
 $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE=59
 $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION=60
 $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61
+$global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62
+$global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63
 
 # Please add new error code for downloading new packages in RP code too
 $global:ErrorCodeNames = @(
@@ -129,7 +131,9 @@ $global:ErrorCodeNames = @(
     "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE",
     "WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE",
     "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION",
-    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE"
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE",
+    "WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG",
+    "WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP"
 )
 
 # NOTE: KubernetesVersion does not contain "v"

diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData
diff --git a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData
diff --git a/pkg/templates/templates_generated.go b/pkg/templates/templates_generated.go
diff --git a/staging/cse/windows/README b/staging/cse/windows/README
@@ -13,8 +13,8 @@
 1. Run below commands to build a test package
 ```bash
 branchName="master"
-currentCseVersion="v0.0.39" # `WindowsCSEScriptsPackage` defined in `parts/windows/kuberneteswindowssetup.ps1`
-testCseVersion="v0.0.39.0" # Test package name. NOTE: Please do not use the official package format and earlier used version.
+currentCseVersion="v0.0.40" # `WindowsCSEScriptsPackage` defined in `parts/windows/kuberneteswindowssetup.ps1`
+testCseVersion="v0.0.40.0" # Test package name. NOTE: Please do not use the official package format and earlier used version.
 url="https://raw.githubusercontent.com/Azure/AgentBaker/$branchName/staging/cse/windows"
 
 mkdir -p temp-work-folder/aks-windows-cse
@@ -55,6 +55,9 @@ popd
 # AKS Windows CSE Scripts Package
 All files except *.test.ps1 and README will be published in AKS Windows CSE Scripts Package.
 
+## v0.0.40
+- feat: set --node-ip for Windows kubelet in k8s v1.29+ #4148
+
 ## v0.0.39
 - feat: support Windows container local dumps #3684
 

diff --git a/staging/cse/windows/azurecnifunc.ps1 b/staging/cse/windows/azurecnifunc.ps1
@@ -403,6 +403,7 @@ function New-ExternalHnsNetwork
     Write-Log "Creating new HNS network `"ext`""
     $externalNetwork = "ext"
     $nas = @(Get-NetAdapter -Physical)
+    $nodeIPs = @()
 
     if ($nas.Count -eq 0) {
         Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST -ErrorMessage "Failed to find any physical network adapters"
@@ -416,6 +417,36 @@ function New-ExternalHnsNetwork
         {
             $managementIP = $netIP.IPAddress
             $adapterName = $na.Name
+
+            Write-Log "Get node IPv4 address assigned to the adapter $($na.Name): $($managementIP)"
+            $nodeIPs += $managementIP
+
+            if ($IsDualStackEnabled) {
+                $netIPv6s = Get-NetIPAddress -ifIndex $na.ifIndex -AddressFamily IPv6 -ErrorAction SilentlyContinue -ErrorVariable netIPErr
+                foreach($ipv6 in $netIPv6s)
+                {
+                    # On an Azure Windows VM, there are two IPv6 IP addresses. Below is an example. It is same in an Azure Linux VM.
+                    # ifIndex IPAddress                                       PrefixLength PrefixOrigin SuffixOrigin AddressState PolicyStore
+                    # ------- ---------                                       ------------ ------------ ------------ ------------ -----------
+                    # 6       fe80::97bd:baf7:2853:f73d%6                               64 WellKnown    Link         Preferred    ActiveStore
+                    # 6       2404:f800:8000:122::4                                    128 Dhcp         Dhcp         Preferred    ActiveStore
+                    #
+                    # From the found docuements. fe80: with WellKnown is the link-local address so we should ignore it.
+                    # IPv6 link-local is a special type of unicast address that is auto-configured on any interface using a combination of 
+                    # the link-local prefix FE80::/10 (first 10 bits equal to 1111 1110 10) and the MAC address of the interface.
+                    #
+                    # https://learn.microsoft.com/en-us/dotnet/api/system.net.networkinformation.prefixorigin?view=net-8.0
+                    # WellKnown | 2 | The prefix is a well-known prefix. Well-known prefixes are specified in standard-track Request for 
+                    # Comments (RFC) documents and assigned by the Internet Assigned Numbers Authority (Iana) or an address registry. Such 
+                    # prefixes are reserved for special purposes. -- | -- | --
+                    if ($ipv6.PrefixOrigin -ne "WellKnown")
+                    {
+                        Write-Log "Get node IPv6 address assigned to the adapter $($na.Name): $($ipv6.IPAddress)"
+                        $nodeIPs += $ipv6.IPAddress
+                    }
+                }
+            }
+
             break
         }
         else {
@@ -432,6 +463,24 @@ function New-ExternalHnsNetwork
         Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP -ErrorMessage "None of the physical network adapters has an IP address"
     }
 
+    # https://github.com/kubernetes/kubernetes/pull/121028
+    if (([version]$global:KubeBinariesVersion).CompareTo([version]("1.29.0")) -ge 0) {
+        Logs-To-Event -TaskName "AKS.WindowsCSE.UpdateKubeClusterConfig" -TaskMessage "Start to update KubeCluster Config. NodeIPs: $nodeIPs"
+
+        # It should always get ipv4 address. Otherwise, it will throw WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP
+        if ($IsDualStackEnabled -and $nodeIPs.Count -eq 1) {
+            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP -ErrorMessage "Failed to get node IPv6 IP address"
+        }
+
+        try {
+            $clusterConfiguration = ConvertFrom-Json ((Get-Content $global:KubeClusterConfigPath -ErrorAction Stop) | Out-String)
+            $clusterConfiguration.Kubernetes.Kubelet.ConfigArgs += "--node-ip=$($nodeIPs -join ',')"
+            $clusterConfiguration | ConvertTo-Json -Depth 10 | Out-File -FilePath $global:KubeClusterConfigPath
+        } catch {
+            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG -ErrorMessage "Failed in updating kube cluster config. Error: $_"
+        }
+    }
+
     Write-Log "Using adapter $adapterName with IP address $managementIP"
     $mgmtIPAfterNetworkCreate
 

diff --git a/vhdbuilder/packer/generate-windows-vhd-configuration.ps1 b/vhdbuilder/packer/generate-windows-vhd-configuration.ps1
@@ -119,7 +119,8 @@ $global:map = @{
     "c:\akse-cache\"              = @(
         "https://acs-mirror.azureedge.net/ccgakvplugin/v1.1.5/binaries/windows-gmsa-ccgakvplugin-v1.1.5.zip",
         "https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.37.zip",
-        "https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.39.zip"
+        "https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.39.zip",
+        "https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.40.zip"
     );
     # Different from other packages which are downloaded/cached and used later only during CSE, windows containerd is installed
     # during building the Windows VHD to cache container images.