Merge pull request #6511 from nilo19/fix/cherry-pick-6136-1-28

[release-1.28] fix: ignore Windows 2019 in ensureVMSSInPool for IPv6 backend pools only
kubernetes-sigs · Jul 8, 2024 · 5d4daf9 · 5d4daf9
2 parents a1ae2d9 + 8dc2055
commit 5d4daf9
Show file tree

Hide file tree

Showing 3 changed files with 160 additions and 1 deletion.
diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go
@@ -121,6 +121,12 @@ const (
 	VMSetCIDRIPV4TagKey = "kubernetesNodeCIDRMaskIPV4"
 	// VMSetCIDRIPV6TagKey specifies the node ipv6 CIDR mask of the instances on the VMSS or VMAS
 	VMSetCIDRIPV6TagKey = "kubernetesNodeCIDRMaskIPV6"
+	// VmssWindows2019ImageGalleryName is the name of Windows 2019 images from the
+	// Microsoft.Compute/galleries/AKSWindows gallery
+	VmssWindows2019ImageGalleryName = "windows-2019-containerd"
+	// Windows2019OSBuildVersion is the official build version of Windows Server 2019
+	// https://learn.microsoft.com/en-us/windows-server/get-started/windows-server-release-info
+	Windows2019OSBuildVersion = "17763"
 
 	// TagsDelimiter is the delimiter of tags
 	TagsDelimiter = ","

diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
@@ -1272,6 +1272,14 @@ func (ss *ScaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back
 			klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration of vmss %s", vmssName)
 			continue
 		}
+
+		// It is possible to run Windows 2019 nodes in IPv4-only mode in a dual-stack cluster. IPv6 is not supported on
+		// Windows 2019 nodes and therefore does not need to be added to the IPv6 backend pool.
+		if isWindows2019(vmss) && isBackendPoolIPv6(backendPoolID) {
+			klog.V(3).Infof("ensureVMSSInPool: vmss %s is Windows 2019, skipping adding to IPv6 backend pool", vmssName)
+			continue
+		}
+
 		vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
 		primaryNIC, err := getPrimaryNetworkInterfaceConfiguration(vmssNIC, vmssName)
 		if err != nil {
@@ -1347,6 +1355,46 @@ func (ss *ScaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back
 	return nil
 }
 
+// isWindows2019 checks if the ImageReference on the VMSS matches a Windows Server 2019 image.
+func isWindows2019(vmss *compute.VirtualMachineScaleSet) bool {
+	if vmss == nil {
+		return false
+	}
+
+	if vmss.VirtualMachineProfile == nil || vmss.VirtualMachineProfile.StorageProfile == nil {
+		return false
+	}
+
+	storageProfile := vmss.VirtualMachineProfile.StorageProfile
+
+	if storageProfile.OsDisk == nil || storageProfile.OsDisk.OsType != compute.OperatingSystemTypesWindows {
+		return false
+	}
+
+	if storageProfile.ImageReference == nil || storageProfile.ImageReference.ID == nil {
+		return false
+	}
+	// example: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/AKS-Windows/providers/Microsoft.Compute/galleries/AKSWindows/images/windows-2019-containerd/versions/17763.5820.240516
+	imageRef := *storageProfile.ImageReference.ID
+	parts := strings.Split(imageRef, "/")
+	if len(parts) < 4 {
+		return false
+	}
+
+	imageName := parts[len(parts)-3]
+	if !strings.EqualFold(imageName, consts.VmssWindows2019ImageGalleryName) {
+		return false
+	}
+
+	osVersion := strings.Split(parts[len(parts)-1], ".")
+	if len(osVersion) != 3 {
+		return false
+	}
+	// Windows Server 2019 is build number 17763
+	// https://learn.microsoft.com/en-us/windows-server/get-started/windows-server-release-info
+	return osVersion[0] == consts.Windows2019OSBuildVersion
+}
+
 func (ss *ScaleSet) ensureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID string, vmSetNameOfLB string) error {
 	mc := metrics.NewMetricContext("services", "vmss_ensure_hosts_in_pool", ss.ResourceGroup, ss.SubscriptionID, getServiceName(service))
 	isOperationSucceeded := false

diff --git a/pkg/provider/azure_vmss_test.go b/pkg/provider/azure_vmss_test.go
@@ -58,6 +58,42 @@ const (
 	testLBBackendpoolID2   = "/subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-2"
 )
 
+// helper enum for setting the OS variant
+// of the VMSS image ref.
+type osVersion int
+
+const (
+	unspecified osVersion = iota
+	windows2019
+	windows2022
+	ubuntu
+)
+
+func buildTestOSSpecificVMSSWithLB(name, namePrefix string, lbBackendpoolIDs []string, os osVersion, ipv6 bool) compute.VirtualMachineScaleSet {
+	vmss := buildTestVMSSWithLB(name, namePrefix, lbBackendpoolIDs, ipv6)
+	switch os {
+	case windows2019:
+		vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile = &compute.VirtualMachineScaleSetStorageProfile{
+			OsDisk: &compute.VirtualMachineScaleSetOSDisk{
+				OsType: compute.OperatingSystemTypesWindows,
+			},
+			ImageReference: &compute.ImageReference{
+				ID: pointer.String("/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/AKS-Windows/providers/Microsoft.Compute/galleries/AKSWindows/images/windows-2019-containerd/versions/17763.5820.240516"),
+			},
+		}
+	case windows2022:
+		vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile = &compute.VirtualMachineScaleSetStorageProfile{
+			OsDisk: &compute.VirtualMachineScaleSetOSDisk{
+				OsType: compute.OperatingSystemTypesWindows,
+			},
+			ImageReference: &compute.ImageReference{
+				ID: pointer.String("/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/AKS-Windows/providers/Microsoft.Compute/galleries/AKSWindows/images/windows-2022-containerd/versions/20348.5820.240516"),
+			},
+		}
+	}
+	return vmss
+}
+
 func buildTestVMSSWithLB(name, namePrefix string, lbBackendpoolIDs []string, ipv6 bool) compute.VirtualMachineScaleSet {
 	lbBackendpoolsV4, lbBackendpoolsV6 := make([]compute.SubResource, 0), make([]compute.SubResource, 0)
 	for _, id := range lbBackendpoolIDs {
@@ -2335,6 +2371,7 @@ func TestEnsureVMSSInPool(t *testing.T) {
 		vmSetName             string
 		clusterIP             string
 		nodes                 []*v1.Node
+		os                    osVersion
 		isBasicLB             bool
 		isVMSSDeallocating    bool
 		isVMSSNilNICConfig    bool
@@ -2437,6 +2474,74 @@ func TestEnsureVMSSInPool(t *testing.T) {
 			expectedPutVMSS: false,
 			expectedErr:     fmt.Errorf("failed to find a primary IP configuration (IPv6=true) for the VMSS VM or VMSS \"vmss\""),
 		},
+		{
+			description: "ensureVMSSInPool should skip Windows 2019 VM for IPv6 backend pool",
+			nodes: []*v1.Node{
+				{
+					Spec: v1.NodeSpec{
+						ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
+					},
+				},
+			},
+			isBasicLB:       false,
+			backendPoolID:   testLBBackendpoolID1 + "-" + "IPv6",
+			clusterIP:       "fd00::e68b",
+			expectedPutVMSS: false,
+			setIPv6Config:   false,
+			expectedErr:     nil,
+			os:              windows2019,
+		},
+		{
+			description: "ensureVMSSInPool should add Windows2019 VM to IPv4 backend pool even if service is IPv6",
+			nodes: []*v1.Node{
+				{
+					Spec: v1.NodeSpec{
+						ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
+					},
+				},
+			},
+			isBasicLB:       false,
+			backendPoolID:   testLBBackendpoolID1,
+			clusterIP:       "fd00::e68b",
+			expectedPutVMSS: true,
+			setIPv6Config:   false,
+			expectedErr:     nil,
+			os:              windows2019,
+		},
+		{
+			description: "ensureVMSSInPool should add Windows 2022 VM to IPv6 backend pool",
+			nodes: []*v1.Node{
+				{
+					Spec: v1.NodeSpec{
+						ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
+					},
+				},
+			},
+			isBasicLB:       false,
+			backendPoolID:   testLBBackendpoolID1 + "-" + "IPv6",
+			clusterIP:       "fd00::e68b",
+			expectedPutVMSS: true,
+			setIPv6Config:   true,
+			expectedErr:     nil,
+			os:              windows2022,
+		},
+		{
+			description: "ensureVMSSInPool should fail if no IPv6 network config - Windows 2022",
+			nodes: []*v1.Node{
+				{
+					Spec: v1.NodeSpec{
+						ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
+					},
+				},
+			},
+			isBasicLB:       false,
+			backendPoolID:   testLBBackendpoolID1 + "-" + "IPv6",
+			clusterIP:       "fd00::e68b",
+			expectedPutVMSS: false,
+			setIPv6Config:   false,
+			expectedErr:     fmt.Errorf("failed to find a primary IP configuration (IPv6=true) for the VMSS VM or VMSS \"vmss\""),
+			os:              windows2022,
+		},
 		{
 			description: "ensureVMSSInPool should update the VMSS correctly for IPv6",
 			nodes: []*v1.Node{
@@ -2521,7 +2626,7 @@ func TestEnsureVMSSInPool(t *testing.T) {
 				ss.LoadBalancerSku = consts.LoadBalancerSkuStandard
 			}
 
-			expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, test.setIPv6Config)
+			expectedVMSS := buildTestOSSpecificVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, test.os, test.setIPv6Config)
 			if test.isVMSSDeallocating {
 				expectedVMSS.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 			}