Merge branch 'master' into andyzhangx/upgrade-disk-driver-v1.30.4

apiserver/apiserver.go

@@ -20,7 +20,7 @@ type OptionConfigurator func(opts *Options)
 // Options holds the options for the api server.
 type Options struct {
 	Addr    string
-	Toggles *toggles.Toggles
+	Toggles toggles.Toggles
 }
 
 func (o *Options) validate() error {

e2e/aks_model.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/Azure/agentbakere2e/config"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
 )
 
@@ -48,6 +48,10 @@ func getBaseClusterModel(clusterName string) *armcontainerservice.ManagedCluster
 					OSDiskSizeGB: to.Ptr[int32](512),
 				},
 			},
+			AutoUpgradeProfile: &armcontainerservice.ManagedClusterAutoUpgradeProfile{
+				NodeOSUpgradeChannel: to.Ptr(armcontainerservice.NodeOSUpgradeChannelNodeImage),
+				UpgradeChannel:       to.Ptr(armcontainerservice.UpgradeChannelNone),
+			},
 			NetworkProfile: &armcontainerservice.NetworkProfile{
 				NetworkPlugin: to.Ptr(armcontainerservice.NetworkPluginKubenet),
 			},

e2e/cluster.go

@@ -16,7 +16,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
 )
 
@@ -39,6 +39,7 @@ type Cluster struct {
 	Kube                           *Kubeclient
 	SubnetID                       string
 	NodeBootstrappingConfiguration *datamodel.NodeBootstrappingConfiguration
+	Maintenance                    *armcontainerservice.MaintenanceConfiguration
 }
 
 // Returns true if the cluster is configured with Azure CNI
@@ -106,6 +107,11 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 		return nil, err
 	}
 
+	maintenance, err := getOrCreateMaintenanceConfiguration(ctx, t, cluster)
+	if err != nil {
+		return nil, fmt.Errorf("get or create maintenance configuration: %w", err)
+	}
+
 	// sometimes tests can be interrupted and vmss are left behind
 	// don't waste resource and delete them
 	if err := collectGarbageVMSS(ctx, t, cluster); err != nil {
@@ -131,7 +137,7 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 		return nil, fmt.Errorf("get node bootstrapping configuration: %w", err)
 	}
 
-	return &Cluster{Model: cluster, Kube: kube, SubnetID: subnetID, NodeBootstrappingConfiguration: nbc}, nil
+	return &Cluster{Model: cluster, Kube: kube, SubnetID: subnetID, NodeBootstrappingConfiguration: nbc, Maintenance: maintenance}, nil
 }
 
 func hash(cluster *armcontainerservice.ManagedCluster) string {
@@ -227,6 +233,50 @@ func createNewAKSClusterWithRetry(ctx context.Context, t *testing.T, cluster *ar
 	return nil, fmt.Errorf("failed to create cluster after %d attempts due to persistent 409 Conflict: %w", maxRetries, lastErr)
 }
 
+func getOrCreateMaintenanceConfiguration(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster) (*armcontainerservice.MaintenanceConfiguration, error) {
+	existingMaintenance, err := config.Azure.Maintenance.Get(ctx, config.ResourceGroupName, *cluster.Name, "default", nil)
+	var azErr *azcore.ResponseError
+	if errors.As(err, &azErr) && azErr.StatusCode == 404 {
+		return createNewMaintenanceConfiguration(ctx, t, cluster)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to get maintenance configuration 'default' for cluster %q: %w", *cluster.Name, err)
+	}
+	return &existingMaintenance.MaintenanceConfiguration, nil
+}
+
+func createNewMaintenanceConfiguration(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster) (*armcontainerservice.MaintenanceConfiguration, error) {
+	t.Logf("creating maintenance configuration for cluster %s in rg %s\n", *cluster.Name, config.ResourceGroupName)
+	maintenance := armcontainerservice.MaintenanceConfiguration{
+		Properties: &armcontainerservice.MaintenanceConfigurationProperties{
+			MaintenanceWindow: &armcontainerservice.MaintenanceWindow{
+				NotAllowedDates: []*armcontainerservice.DateSpan{ // no maintenance till 2100
+					{
+						End:   to.Ptr(func() time.Time { t, _ := time.Parse("2006-01-02", "2100-01-01"); return t }()),
+						Start: to.Ptr(func() time.Time { t, _ := time.Parse("2006-01-02", "2000-01-01"); return t }()),
+					}},
+				DurationHours: to.Ptr[int32](4),
+				StartTime:     to.Ptr("00:00"),  //PST
+				UTCOffset:     to.Ptr("+08:00"), //PST
+				Schedule: &armcontainerservice.Schedule{
+					RelativeMonthly: &armcontainerservice.RelativeMonthlySchedule{
+						DayOfWeek:      to.Ptr(armcontainerservice.WeekDayMonday),
+						IntervalMonths: to.Ptr[int32](3),
+						WeekIndex:      to.Ptr(armcontainerservice.TypeFirst),
+					},
+				},
+			},
+		},
+	}
+
+	_, err := config.Azure.Maintenance.CreateOrUpdate(ctx, config.ResourceGroupName, *cluster.Name, "default", maintenance, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create maintenance configuration: %w", err)
+	}
+
+	return &maintenance, nil
+}
+
 type VNet struct {
 	name     string
 	subnetId string

e2e/config/azure.go

@@ -13,7 +13,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
 	"github.com/Azure/go-armbalancer"
@@ -27,6 +27,7 @@ type AzureClient struct {
 	Resource                  *armresources.Client
 	ResourceGroup             *armresources.ResourceGroupsClient
 	AKS                       *armcontainerservice.ManagedClustersClient
+	Maintenance               *armcontainerservice.MaintenanceConfigurationsClient
 	SecurityGroup             *armnetwork.SecurityGroupsClient
 	Subnet                    *armnetwork.SubnetsClient
 	GalleryImageVersionClient *armcompute.GalleryImageVersionsClient
@@ -118,6 +119,11 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 		return nil, fmt.Errorf("failed to create aks client: %w", err)
 	}
 
+	cloud.Maintenance, err = armcontainerservice.NewMaintenanceConfigurationsClient(subscription, credential, opts)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create maintenance client: %w", err)
+	}
+
 	cloud.VMSS, err = armcompute.NewVirtualMachineScaleSetsClient(subscription, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create vmss client: %w", err)

e2e/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
 	github.com/Azure/go-armbalancer v0.0.2

e2e/go.sum

@@ -8,6 +8,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0/go.mod h1:TmlMW4W5OvXOmOyKNnor8nlMMiO1ctIyzmHme/VHsrA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0 h1:EK0ZY1qKWzaWyRNFDsrwRfgVBMGbs+m71yie+y11+Tc=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0/go.mod h1:drbnYtukMoZqUQq9hJASf41w3RB4VoTJPoPpe+XDHPU=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw=

e2e/nodebootstrapping.go

@@ -18,13 +18,13 @@ import (
 func getNodeBootstrapping(ctx context.Context, nbc *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) {
 	switch e2eMode {
 	case "coverage":
-		return getNodeBootstrappingForCoverage(ctx, nbc)
+		return getNodeBootstrappingForCoverage(nbc)
 	default:
 		return getNodeBootstrappingForValidation(ctx, nbc)
 	}
 }
 
-func getNodeBootstrappingForCoverage(ctx context.Context, nbc *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) {
+func getNodeBootstrappingForCoverage(nbc *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) {
 	payload, err := json.Marshal(nbc)
 	if err != nil {
 		log.Fatalf("failed to marshal nbc, error: %s", err)

e2e/pod.go

@@ -25,14 +25,15 @@ func getHostNetworkDebugPodName(ctx context.Context, kube *Kubeclient) (string,
 	if err := kube.Dynamic.List(ctx, &podList, client.MatchingLabels{"app": hostNetworkDebugAppLabel}); err != nil {
 		return "", fmt.Errorf("failed to list debug pod: %w", err)
 	}
-	for _, pod := range podList.Items {
-		err := waitUntilPodReady(ctx, kube, pod.Name)
-		if err != nil {
-			return "", fmt.Errorf("failed to wait for pod to be in running state: %w", err)
-		}
-		return pod.Name, nil
+	if podList.Size() == 0 {
+		return "", fmt.Errorf("failed to find host debug pod")
+	}
+	pod := podList.Items[0]
+	err := waitUntilPodReady(ctx, kube, pod.Name)
+	if err != nil {
+		return "", fmt.Errorf("failed to wait for pod to be in running state: %w", err)
 	}
-	return "", fmt.Errorf("failed to find non host debug pod")
+	return pod.Name, nil
 }
 
 // Returns the name of a pod that's a member of the 'debugnonhost' daemonset running in the cluster - this will return

e2e/pollers.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/Azure/agentbakere2e/config"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

e2e/scenario_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/Azure/agentbakere2e/toolkit"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 )
 
 func Test_azurelinuxv2(t *testing.T) {

e2e/validators.go

@@ -27,7 +27,7 @@ func DirectoryValidator(path string, files []string) *LiveVMValidator {
 
 func SysctlConfigValidator(customSysctls map[string]string) *LiveVMValidator {
 	keysToCheck := make([]string, len(customSysctls))
-	for k, _ := range customSysctls {
+	for k := range customSysctls {
 		keysToCheck = append(keysToCheck, k)
 	}
 	// regex used in sed command to remove extra spaces between two numerical values, used to verify correct values for
@@ -199,7 +199,7 @@ func ServiceCanRestartValidator(serviceName string, restartTimeoutInSeconds int)
 		Command:     command,
 		Asserter: func(code, stdout, stderr string) error {
 			if code != "0" {
-				return fmt.Errorf("service kill and check terminated with exit code %q (expected 0).\nCommand: %s\n\nStdout:\n%s\n\n Stderr:\n%s\n", code, command, stdout, stderr)
+				return fmt.Errorf("service kill and check terminated with exit code %q (expected 0).\nCommand: %s\n\nStdout:\n%s\n\n Stderr:\n%s", code, command, stdout, stderr)
 			}
 			return nil
 		},
@@ -209,7 +209,7 @@ func ServiceCanRestartValidator(serviceName string, restartTimeoutInSeconds int)
 func CommandHasOutputValidator(commandToExecute string, expectedOutput string) *LiveVMValidator {
 	steps := []string{
 		// Verify the service is active - print the state then verify so we have logs
-		fmt.Sprintf("%s", commandToExecute),
+		fmt.Sprint(commandToExecute),
 	}
 
 	command := makeExecutableCommand(steps)
@@ -219,10 +219,10 @@ func CommandHasOutputValidator(commandToExecute string, expectedOutput string) *
 		Command:     command,
 		Asserter: func(code, stdout, stderr string) error {
 			if !strings.Contains(stderr, expectedOutput) {
-				return fmt.Errorf("'%s' output did not contain expected string Stdout:\n%s\n\n Stderr:\n%s\n", command, stdout, stderr)
+				return fmt.Errorf("'%s' output did not contain expected string Stdout:\n%s\n\n Stderr:\n%s", command, stdout, stderr)
 			}
 			if code != "0" {
-				return fmt.Errorf("command failed with exit code %q (expected 0).\nCommand: %s\n\nStdout:\n%s\n\n Stderr:\n%s\n", code, command, stdout, stderr)
+				return fmt.Errorf("command failed with exit code %q (expected 0).\nCommand: %s\n\nStdout:\n%s\n\n Stderr:\n%s", code, command, stdout, stderr)
 			}
 			return nil
 		},
@@ -301,12 +301,12 @@ func kubeletNodeIPValidator() *LiveVMValidator {
 			// Search for "--node-ip" flag and its value.
 			matches := regexp.MustCompile(`--node-ip=([a-zA-Z0-9.,]*)`).FindStringSubmatch(stdout)
 			if matches == nil || len(matches) < 2 {
-				return fmt.Errorf("Could not find kubelet flag --node-ip")
+				return fmt.Errorf("could not find kubelet flag --node-ip")
 			}
 
 			ipAddresses := strings.Split(matches[1], ",") // Could be multiple for dual-stack.
 			if len(ipAddresses) == 0 || len(ipAddresses) > 2 {
-				return fmt.Errorf("Expected one or two --node-ip addresses, but got %d", len(ipAddresses))
+				return fmt.Errorf("expected one or two --node-ip addresses, but got %d", len(ipAddresses))
 			}
 
 			// Check that each IP is a valid address.

e2e/vmss.go

@@ -40,7 +40,7 @@ func createVMSS(ctx context.Context, t *testing.T, vmssName string, opts *scenar
 	require.NoError(t, err, vmssName, opts)
 
 	if isAzureCNI {
-		err = addPodIPConfigsForAzureCNI(ctx, &model, vmssName, opts)
+		err = addPodIPConfigsForAzureCNI(&model, vmssName, opts)
 		require.NoError(t, err)
 	}
 
@@ -115,7 +115,7 @@ func deleteVMSS(t *testing.T, ctx context.Context, vmssName string, opts *scenar
 // Adds additional IP configs to the passed in vmss model based on the chosen cluster's setting of "maxPodsPerNode",
 // as we need be able to allow AKS to allocate an additional IP config for each pod running on the given node.
 // Additional info: https://learn.microsoft.com/en-us/azure/aks/configure-azure-cni
-func addPodIPConfigsForAzureCNI(ctx context.Context, vmss *armcompute.VirtualMachineScaleSet, vmssName string, opts *scenarioRunOpts) error {
+func addPodIPConfigsForAzureCNI(vmss *armcompute.VirtualMachineScaleSet, vmssName string, opts *scenarioRunOpts) error {
 	maxPodsPerNode, err := opts.clusterConfig.MaxPodsPerNode()
 	if err != nil {
 		return fmt.Errorf("failed to read agentpool MaxPods value from chosen cluster model: %w", err)