Merge branch 'master' into andyzhangx/upgrade-file-driver-v1.31.3

e2e/exec.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/Azure/agentbaker/e2e/config"
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -97,29 +98,80 @@ func sshKeyName(vmPrivateIP string) string {
 }
 
 func sshString(vmPrivateIP string) string {
-	return fmt.Sprintf(`ssh -i %s -o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 azureuser@%s`, sshKeyName(vmPrivateIP), vmPrivateIP)
+	return fmt.Sprintf(`ssh -i %[1]s -o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 azureuser@%[2]s`, sshKeyName(vmPrivateIP), vmPrivateIP)
 }
 
-func execOnVM(ctx context.Context, kube *Kubeclient, vmPrivateIP, jumpboxPodName, sshPrivateKey, command string) (*podExecResult, error) {
-	sshCommand := fmt.Sprintf(`echo '%s' > %[2]s && chmod 0600 %[2]s && %s`, sshPrivateKey, sshKeyName(vmPrivateIP), sshString(vmPrivateIP))
-	sshCommand = sshCommand + " sudo"
-	commandToExecute := fmt.Sprintf("%s %s", sshCommand, command)
+func quoteForBash(command string) string {
+	return fmt.Sprintf("'%s'", strings.ReplaceAll(command, "'", "'\"'\"'"))
+}
+
+func execBashCommandOnVM(ctx context.Context, s *Scenario, vmPrivateIP, jumpboxPodName, sshPrivateKey, command string) (*podExecResult, error) {
+	image := &config.Image{
+		OS: config.OSUbuntu,
+	}
+	return execScriptOnVm(ctx, s, vmPrivateIP, jumpboxPodName, sshPrivateKey, []string{command}, image)
+}
+
+func execScriptOnVm(ctx context.Context, s *Scenario, vmPrivateIP, jumpboxPodName, sshPrivateKey string, script []string, os *config.Image) (*podExecResult, error) {
+	/*
+		This works in a way that doesn't rely on the node having joined the cluster:
+		* We create a linux pod on a different node.
+		* on that pod, we create a script file containing the script passed into this method.
+		* Then we scp the script to the node under test.
+		* Then we execute the script using an interpreter (powershell or bash) based on the OS of the node.
+	*/
+	identifier := uuid.New().String()
+
+	var scriptFileName, remoteScriptFileName string
+	if os.OS == config.OSWindows {
+		scriptFileName = fmt.Sprintf("script_file_%s.ps1", identifier)
+		remoteScriptFileName = fmt.Sprintf("c:/%s", scriptFileName)
+	} else {
+		scriptFileName = fmt.Sprintf("script_file_%s.sh", identifier)
+		remoteScriptFileName = scriptFileName
+	}
+
+	var interpreter string
+	switch os.OS {
+	case config.OSWindows:
+		interpreter = "powershell"
+		break
+	default:
+		interpreter = "bash"
+		break
+	}
+
+	scriptWithLineBreaks := strings.Join(script, "\n")
+	steps := []string{
+		fmt.Sprintf("echo '%[1]s' > %[2]s", sshPrivateKey, sshKeyName(vmPrivateIP)),
+		"set -x",
+		fmt.Sprintf("echo %[1]s > %[2]s", quoteForBash(scriptWithLineBreaks), scriptFileName),
+		fmt.Sprintf("chmod 0600 %s", sshKeyName(vmPrivateIP)),
+		fmt.Sprintf("chmod 0755 %s", scriptFileName),
+		fmt.Sprintf(`scp -i %[1]s -o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 %[3]s azureuser@%[2]s:%[4]s`, sshKeyName(vmPrivateIP), vmPrivateIP, scriptFileName, remoteScriptFileName),
+		fmt.Sprintf("%s %s %s", sshString(vmPrivateIP), interpreter, remoteScriptFileName),
+	}
+
+	joinedSteps := strings.Join(steps, " && ")
+
+	s.T.Log(fmt.Sprintf("Executing script %s:\n---START-SCRIPT---\n%s\n---END-SCRIPT---\n", scriptFileName, scriptWithLineBreaks))
 
-	execResult, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, jumpboxPodName, commandToExecute)
+	kube := s.Runtime.Cluster.Kube
+	execResult, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, jumpboxPodName, joinedSteps)
 	if err != nil {
 		return nil, fmt.Errorf("error executing command on pod: %w", err)
 	}
 
 	return execResult, nil
 }
 
-func execOnPrivilegedPod(ctx context.Context, kube *Kubeclient, namespace, podName string, command string) (*podExecResult, error) {
-	privilegedCommand := append(privelegedCommandArray(), command)
+func execOnPrivilegedPod(ctx context.Context, kube *Kubeclient, namespace string, podName string, bashCommand string) (*podExecResult, error) {
+	privilegedCommand := append(privilegedCommandArray(), bashCommand)
 	return execOnPod(ctx, kube, namespace, podName, privilegedCommand)
 }
 
-func execOnUnprivilegedPod(ctx context.Context, kube *Kubeclient, namespace, podName, command string) (*podExecResult, error) {
-	nonPrivilegedCommand := append(unprivilegedCommandArray(), command)
+func execOnUnprivilegedPod(ctx context.Context, kube *Kubeclient, namespace string, podName string, bashCommand string) (*podExecResult, error) {
+	nonPrivilegedCommand := append(unprivilegedCommandArray(), bashCommand)
 	return execOnPod(ctx, kube, namespace, podName, nonPrivilegedCommand)
 }
 
@@ -170,7 +222,7 @@ func execOnPod(ctx context.Context, kube *Kubeclient, namespace, podName string,
 	}, nil
 }
 
-func privelegedCommandArray() []string {
+func privilegedCommandArray() []string {
 	return []string{
 		"chroot",
 		"/proc/1/root",

e2e/scenario_helpers_test.go

@@ -122,6 +122,18 @@ func prepareAKSNode(ctx context.Context, s *Scenario) {
 	}
 	var err error
 	s.Runtime.SSHKeyPrivate, s.Runtime.SSHKeyPublic, err = getNewRSAKeyPair()
+	publicKeyData := datamodel.PublicKey{KeyData: string(s.Runtime.SSHKeyPublic)}
+
+	// check it all.
+	if s.Runtime.NBC != nil && s.Runtime.NBC.ContainerService != nil && s.Runtime.NBC.ContainerService.Properties != nil && s.Runtime.NBC.ContainerService.Properties.LinuxProfile != nil {
+		if s.Runtime.NBC.ContainerService.Properties.LinuxProfile.SSH.PublicKeys == nil {
+			s.Runtime.NBC.ContainerService.Properties.LinuxProfile.SSH.PublicKeys = []datamodel.PublicKey{}
+		}
+		// Windows fetches SSH keys from the linux profile and replaces any existing SSH keys with these. So we have to set
+		// the Linux SSH keys for Windows SSH to work. Yeah. I find it odd too.
+		s.Runtime.NBC.ContainerService.Properties.LinuxProfile.SSH.PublicKeys = append(s.Runtime.NBC.ContainerService.Properties.LinuxProfile.SSH.PublicKeys, publicKeyData)
+	}
+
 	require.NoError(s.T, err)
 	createVMSS(ctx, s)
 	err = getCustomScriptExtensionStatus(ctx, s)
@@ -179,7 +191,7 @@ func validateVM(ctx context.Context, s *Scenario) {
 
 	// skip when outbound type is block as the wasm will create pod from gcr, however, network isolated cluster scenario will block egress traffic of gcr.
 	// TODO(xinhl): add another way to validate
-	if s.Runtime.NBC != nil && s.Runtime.NBC.AgentPoolProfile.WorkloadRuntime == datamodel.WasmWasi && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeBlock && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeNone {
+	if s.Runtime.NBC != nil && s.Runtime.NBC.AgentPoolProfile != nil && s.Runtime.NBC.AgentPoolProfile.WorkloadRuntime == datamodel.WasmWasi && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeBlock && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeNone {
 		ValidateWASM(ctx, s, s.Runtime.KubeNodeName)
 	}
 	if s.Runtime.AKSNodeConfig != nil && s.Runtime.AKSNodeConfig.WorkloadRuntime == aksnodeconfigv1.WorkloadRuntime_WORKLOAD_RUNTIME_WASM_WASI {

e2e/scenario_win_test.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"context"
 	"testing"
 
 	"github.com/Azure/agentbaker/e2e/config"
@@ -16,6 +17,10 @@ func Test_Windows2019Containerd(t *testing.T) {
 			VHD:                    config.VHDWindows2019Containerd,
 			VMConfigMutator:        func(vmss *armcompute.VirtualMachineScaleSet) {},
 			BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) {},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote")
+				ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"})
+			},
 		},
 	})
 }
@@ -28,7 +33,10 @@ func Test_Windows2022Containerd(t *testing.T) {
 			VHD:             config.VHDWindows2022Containerd,
 			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {},
 			BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) {
-
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote")
+				ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"})
 			},
 		},
 	})
@@ -42,7 +50,10 @@ func Test_Windows2022ContainerdGen2(t *testing.T) {
 			VHD:             config.VHDWindows2022ContainerdGen2,
 			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {},
 			BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) {
-
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote")
+				ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"})
 			},
 		},
 	})
@@ -56,7 +67,10 @@ func Test_Windows23H2(t *testing.T) {
 			VHD:             config.VHDWindows23H2,
 			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {},
 			BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) {
-
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote")
+				ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"})
 			},
 		},
 	})
@@ -70,7 +84,10 @@ func Test_Windows23H2Gen2(t *testing.T) {
 			VHD:             config.VHDWindows23H2Gen2,
 			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {},
 			BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) {
-
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote")
+				ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"})
 			},
 		},
 	})

e2e/validation.go

@@ -34,11 +34,12 @@ func ValidateWASM(ctx context.Context, s *Scenario, nodeName string) {
 }
 
 func ValidateCommonLinux(ctx context.Context, s *Scenario) {
-	execResult := execOnVMForScenarioValidateExitCode(ctx, s, "cat /etc/default/kubelet", 0, "could not read kubelet config")
-	require.NotContains(s.T, execResult.stdout.String(), "--dynamic-config-dir", "kubelet flag '--dynamic-config-dir' should not be present in /etc/default/kubelet")
+	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{"sudo cat /etc/default/kubelet"}, 0, "could not read kubelet config")
+	stdout := execResult.stdout.String()
+	require.NotContains(s.T, stdout, "--dynamic-config-dir", "kubelet flag '--dynamic-config-dir' should not be present in /etc/default/kubelet\nContents:\n%s")
 
 	// the instructions belows expects the SSH key to be uploaded to the user pool VM.
-	// which happens as a side-effect of execOnVMForScenario, it's ugly but works.
+	// which happens as a side-effect of execCommandOnVMForScenario, it's ugly but works.
 	// maybe we should use a single ssh key per cluster, but need to be careful with parallel test runs.
 	logSSHInstructions(s)
 
@@ -61,7 +62,7 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) {
 		//"cloud-config.txt", // file with UserData
 	})
 
-	execResult = execOnVMForScenarioValidateExitCode(ctx, s, "curl http://168.63.129.16:32526/vmSettings", 0, "curl to wireserver failed")
+	execResult = execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{"sudo curl http://168.63.129.16:32526/vmSettings"}, 0, "curl to wireserver failed")
 
 	execResult = execOnVMForScenarioOnUnprivilegedPod(ctx, s, "curl https://168.63.129.16/machine/?comp=goalstate -H 'x-ms-version: 2015-04-05' -s --connect-timeout 4")
 	require.Equal(s.T, "28", execResult.exitCode, "curl to wireserver should fail")