diff --git a/e2e/exec.go b/e2e/exec.go index 4c6db305fef..5b073ab6bd3 100644 --- a/e2e/exec.go +++ b/e2e/exec.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/Azure/agentbaker/e2e/config" + "github.com/google/uuid" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -97,15 +98,66 @@ func sshKeyName(vmPrivateIP string) string { } func sshString(vmPrivateIP string) string { - return fmt.Sprintf(`ssh -i %s -o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 azureuser@%s`, sshKeyName(vmPrivateIP), vmPrivateIP) + return fmt.Sprintf(`ssh -i %[1]s -o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 azureuser@%[2]s`, sshKeyName(vmPrivateIP), vmPrivateIP) } -func execOnVM(ctx context.Context, kube *Kubeclient, vmPrivateIP, jumpboxPodName, sshPrivateKey, command string) (*podExecResult, error) { - sshCommand := fmt.Sprintf(`echo '%s' > %[2]s && chmod 0600 %[2]s && %s`, sshPrivateKey, sshKeyName(vmPrivateIP), sshString(vmPrivateIP)) - sshCommand = sshCommand + " sudo" - commandToExecute := fmt.Sprintf("%s %s", sshCommand, command) +func quoteForBash(command string) string { + return fmt.Sprintf("'%s'", strings.ReplaceAll(command, "'", "'\"'\"'")) +} + +func execBashCommandOnVM(ctx context.Context, s *Scenario, vmPrivateIP, jumpboxPodName, sshPrivateKey, command string) (*podExecResult, error) { + image := &config.Image{ + OS: config.OSUbuntu, + } + return execScriptOnVm(ctx, s, vmPrivateIP, jumpboxPodName, sshPrivateKey, []string{command}, image) +} + +func execScriptOnVm(ctx context.Context, s *Scenario, vmPrivateIP, jumpboxPodName, sshPrivateKey string, script []string, os *config.Image) (*podExecResult, error) { + /* + This works in a way that doesn't rely on the node having joined the cluster: + * We create a linux pod on a different node. + * on that pod, we create a script file containing the script passed into this method. + * Then we scp the script to the node under test. + * Then we execute the script using an interpreter (powershell or bash) based on the OS of the node. + */ + identifier := uuid.New().String() + + var scriptFileName, remoteScriptFileName string + if os.OS == config.OSWindows { + scriptFileName = fmt.Sprintf("script_file_%s.ps1", identifier) + remoteScriptFileName = fmt.Sprintf("c:/%s", scriptFileName) + } else { + scriptFileName = fmt.Sprintf("script_file_%s.sh", identifier) + remoteScriptFileName = scriptFileName + } + + var interpreter string + switch os.OS { + case config.OSWindows: + interpreter = "powershell" + break + default: + interpreter = "bash" + break + } + + scriptWithLineBreaks := strings.Join(script, "\n") + steps := []string{ + fmt.Sprintf("echo '%[1]s' > %[2]s", sshPrivateKey, sshKeyName(vmPrivateIP)), + "set -x", + fmt.Sprintf("echo %[1]s > %[2]s", quoteForBash(scriptWithLineBreaks), scriptFileName), + fmt.Sprintf("chmod 0600 %s", sshKeyName(vmPrivateIP)), + fmt.Sprintf("chmod 0755 %s", scriptFileName), + fmt.Sprintf(`scp -i %[1]s -o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 %[3]s azureuser@%[2]s:%[4]s`, sshKeyName(vmPrivateIP), vmPrivateIP, scriptFileName, remoteScriptFileName), + fmt.Sprintf("%s %s %s", sshString(vmPrivateIP), interpreter, remoteScriptFileName), + } + + joinedSteps := strings.Join(steps, " && ") + + s.T.Log(fmt.Sprintf("Executing script %s:\n---START-SCRIPT---\n%s\n---END-SCRIPT---\n", scriptFileName, scriptWithLineBreaks)) - execResult, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, jumpboxPodName, commandToExecute) + kube := s.Runtime.Cluster.Kube + execResult, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, jumpboxPodName, joinedSteps) if err != nil { return nil, fmt.Errorf("error executing command on pod: %w", err) } @@ -113,13 +165,13 @@ func execOnVM(ctx context.Context, kube *Kubeclient, vmPrivateIP, jumpboxPodName return execResult, nil } -func execOnPrivilegedPod(ctx context.Context, kube *Kubeclient, namespace, podName string, command string) (*podExecResult, error) { - privilegedCommand := append(privelegedCommandArray(), command) +func execOnPrivilegedPod(ctx context.Context, kube *Kubeclient, namespace string, podName string, bashCommand string) (*podExecResult, error) { + privilegedCommand := append(privilegedCommandArray(), bashCommand) return execOnPod(ctx, kube, namespace, podName, privilegedCommand) } -func execOnUnprivilegedPod(ctx context.Context, kube *Kubeclient, namespace, podName, command string) (*podExecResult, error) { - nonPrivilegedCommand := append(unprivilegedCommandArray(), command) +func execOnUnprivilegedPod(ctx context.Context, kube *Kubeclient, namespace string, podName string, bashCommand string) (*podExecResult, error) { + nonPrivilegedCommand := append(unprivilegedCommandArray(), bashCommand) return execOnPod(ctx, kube, namespace, podName, nonPrivilegedCommand) } @@ -170,7 +222,7 @@ func execOnPod(ctx context.Context, kube *Kubeclient, namespace, podName string, }, nil } -func privelegedCommandArray() []string { +func privilegedCommandArray() []string { return []string{ "chroot", "/proc/1/root", diff --git a/e2e/scenario_helpers_test.go b/e2e/scenario_helpers_test.go index ddee8bb5491..86e77c7c9dd 100644 --- a/e2e/scenario_helpers_test.go +++ b/e2e/scenario_helpers_test.go @@ -122,6 +122,18 @@ func prepareAKSNode(ctx context.Context, s *Scenario) { } var err error s.Runtime.SSHKeyPrivate, s.Runtime.SSHKeyPublic, err = getNewRSAKeyPair() + publicKeyData := datamodel.PublicKey{KeyData: string(s.Runtime.SSHKeyPublic)} + + // check it all. + if s.Runtime.NBC != nil && s.Runtime.NBC.ContainerService != nil && s.Runtime.NBC.ContainerService.Properties != nil && s.Runtime.NBC.ContainerService.Properties.LinuxProfile != nil { + if s.Runtime.NBC.ContainerService.Properties.LinuxProfile.SSH.PublicKeys == nil { + s.Runtime.NBC.ContainerService.Properties.LinuxProfile.SSH.PublicKeys = []datamodel.PublicKey{} + } + // Windows fetches SSH keys from the linux profile and replaces any existing SSH keys with these. So we have to set + // the Linux SSH keys for Windows SSH to work. Yeah. I find it odd too. + s.Runtime.NBC.ContainerService.Properties.LinuxProfile.SSH.PublicKeys = append(s.Runtime.NBC.ContainerService.Properties.LinuxProfile.SSH.PublicKeys, publicKeyData) + } + require.NoError(s.T, err) createVMSS(ctx, s) err = getCustomScriptExtensionStatus(ctx, s) @@ -179,7 +191,7 @@ func validateVM(ctx context.Context, s *Scenario) { // skip when outbound type is block as the wasm will create pod from gcr, however, network isolated cluster scenario will block egress traffic of gcr. // TODO(xinhl): add another way to validate - if s.Runtime.NBC != nil && s.Runtime.NBC.AgentPoolProfile.WorkloadRuntime == datamodel.WasmWasi && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeBlock && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeNone { + if s.Runtime.NBC != nil && s.Runtime.NBC.AgentPoolProfile != nil && s.Runtime.NBC.AgentPoolProfile.WorkloadRuntime == datamodel.WasmWasi && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeBlock && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeNone { ValidateWASM(ctx, s, s.Runtime.KubeNodeName) } if s.Runtime.AKSNodeConfig != nil && s.Runtime.AKSNodeConfig.WorkloadRuntime == aksnodeconfigv1.WorkloadRuntime_WORKLOAD_RUNTIME_WASM_WASI { diff --git a/e2e/scenario_win_test.go b/e2e/scenario_win_test.go index ed2454a3daf..86b0573ab4a 100644 --- a/e2e/scenario_win_test.go +++ b/e2e/scenario_win_test.go @@ -1,6 +1,7 @@ package e2e import ( + "context" "testing" "github.com/Azure/agentbaker/e2e/config" @@ -16,6 +17,10 @@ func Test_Windows2019Containerd(t *testing.T) { VHD: config.VHDWindows2019Containerd, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {}, BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) {}, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") + ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) + }, }, }) } @@ -28,7 +33,10 @@ func Test_Windows2022Containerd(t *testing.T) { VHD: config.VHDWindows2022Containerd, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {}, BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) { - + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") + ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) }, }, }) @@ -42,7 +50,10 @@ func Test_Windows2022ContainerdGen2(t *testing.T) { VHD: config.VHDWindows2022ContainerdGen2, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {}, BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) { - + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") + ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) }, }, }) @@ -56,7 +67,10 @@ func Test_Windows23H2(t *testing.T) { VHD: config.VHDWindows23H2, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {}, BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) { - + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") + ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) }, }, }) @@ -70,7 +84,10 @@ func Test_Windows23H2Gen2(t *testing.T) { VHD: config.VHDWindows23H2Gen2, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {}, BootstrapConfigMutator: func(configuration *datamodel.NodeBootstrappingConfiguration) { - + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") + ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) }, }, }) diff --git a/e2e/validation.go b/e2e/validation.go index d254cc1f658..bc112ca65a5 100644 --- a/e2e/validation.go +++ b/e2e/validation.go @@ -34,11 +34,12 @@ func ValidateWASM(ctx context.Context, s *Scenario, nodeName string) { } func ValidateCommonLinux(ctx context.Context, s *Scenario) { - execResult := execOnVMForScenarioValidateExitCode(ctx, s, "cat /etc/default/kubelet", 0, "could not read kubelet config") - require.NotContains(s.T, execResult.stdout.String(), "--dynamic-config-dir", "kubelet flag '--dynamic-config-dir' should not be present in /etc/default/kubelet") + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{"sudo cat /etc/default/kubelet"}, 0, "could not read kubelet config") + stdout := execResult.stdout.String() + require.NotContains(s.T, stdout, "--dynamic-config-dir", "kubelet flag '--dynamic-config-dir' should not be present in /etc/default/kubelet\nContents:\n%s") // the instructions belows expects the SSH key to be uploaded to the user pool VM. - // which happens as a side-effect of execOnVMForScenario, it's ugly but works. + // which happens as a side-effect of execCommandOnVMForScenario, it's ugly but works. // maybe we should use a single ssh key per cluster, but need to be careful with parallel test runs. logSSHInstructions(s) @@ -61,7 +62,7 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) { //"cloud-config.txt", // file with UserData }) - execResult = execOnVMForScenarioValidateExitCode(ctx, s, "curl http://168.63.129.16:32526/vmSettings", 0, "curl to wireserver failed") + execResult = execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{"sudo curl http://168.63.129.16:32526/vmSettings"}, 0, "curl to wireserver failed") execResult = execOnVMForScenarioOnUnprivilegedPod(ctx, s, "curl https://168.63.129.16/machine/?comp=goalstate -H 'x-ms-version: 2015-04-05' -s --connect-timeout 4") require.Equal(s.T, "28", execResult.exitCode, "curl to wireserver should fail") diff --git a/e2e/validators.go b/e2e/validators.go index 22159bbfddb..5e84f275726 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -18,10 +18,14 @@ import ( ) func ValidateDirectoryContent(ctx context.Context, s *Scenario, path string, files []string) { - command := fmt.Sprintf("ls -la %s", path) - execResult := execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not get directory contents") + command := []string{ + "set -ex", + fmt.Sprintf("sudo ls -la %s", path), + } + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not get directory contents") + stdout := execResult.stdout.String() for _, file := range files { - require.Contains(s.T, execResult.stdout.String(), file, "expected to find file %s within directory %s, but did not", file, path) + require.Contains(s.T, stdout, file, "expected to find file %s within directory %s, but did not.\nDirectory contents:\n%s", file, path, stdout) } } @@ -30,81 +34,85 @@ func ValidateSysctlConfig(ctx context.Context, s *Scenario, customSysctls map[st for k := range customSysctls { keysToCheck = append(keysToCheck, k) } - execResult := execOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("sysctl %s | sed -E 's/([0-9])\\s+([0-9])/\\1 \\2/g'", strings.Join(keysToCheck, " ")), 0, "systmctl command failed") + command := []string{ + "set -ex", + fmt.Sprintf("sudo sysctl %s | sed -E 's/([0-9])\\s+([0-9])/\\1 \\2/g'", strings.Join(keysToCheck, " ")), + } + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "systmctl command failed") + stdout := execResult.stdout.String() for name, value := range customSysctls { - require.Contains(s.T, execResult.stdout.String(), fmt.Sprintf("%s = %v", name, value), "expected to find %s set to %v, but was not", name, value) + require.Contains(s.T, stdout, fmt.Sprintf("%s = %v", name, value), "expected to find %s set to %v, but was not.\nStdout:\n%s", name, value, stdout) } } func ValidateNvidiaSMINotInstalled(ctx context.Context, s *Scenario) { - command := "nvidia-smi" - execResult := execOnVMForScenarioValidateExitCode(ctx, s, command, 1, "") - require.Contains(s.T, execResult.stderr.String(), "nvidia-smi: command not found", "expected stderr to contain 'nvidia-smi: command not found', but got %q", execResult.stderr.String()) + command := []string{ + "set -ex", + "sudo nvidia-smi", + } + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 1, "") + stderr := execResult.stderr.String() + require.Contains(s.T, stderr, "nvidia-smi: command not found", "expected stderr to contain 'nvidia-smi: command not found', but got %q", stderr) } func ValidateNvidiaSMIInstalled(ctx context.Context, s *Scenario) { - command := "nvidia-smi" - execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not execute nvidia-smi command") + command := []string{"set -ex", "sudo nvidia-smi"} + execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not execute nvidia-smi command") } func ValidateNvidiaModProbeInstalled(ctx context.Context, s *Scenario) { - command := "nvidia-modprobe" - execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "cound not execute nvidia-modprobe command") + command := []string{ + "set -ex", + "sudo nvidia-modprobe", + } + execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "cound not execute nvidia-modprobe command") } func ValidateNonEmptyDirectory(ctx context.Context, s *Scenario, dirName string) { - command := fmt.Sprintf("ls -1q %s | grep -q '^.*$' && true || false", dirName) - execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "either could not find expected file, or something went wrong") + command := []string{ + "set -ex", + fmt.Sprintf("sudo ls -1q %s | grep -q '^.*$' && true || false", dirName), + } + execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "either could not find expected file, or something went wrong") } func ValidateFileHasContent(ctx context.Context, s *Scenario, fileName string, contents string) { - steps := []string{ - fmt.Sprintf("ls -la %[1]s", fileName), - fmt.Sprintf("sudo cat %[1]s", fileName), - fmt.Sprintf("(sudo cat %[1]s | grep -q -F -e %[2]q)", fileName, contents), - } + if s.VHD.OS == config.OSWindows { + steps := []string{ + fmt.Sprintf("dir %[1]s", fileName), + fmt.Sprintf("Get-Content %[1]s", fileName), + fmt.Sprintf("if (Select-String -Path %s -Pattern \"%s\" -SimpleMatch -Quiet) { return 1 } else { return 0 }", fileName, contents), + } + + execScriptOnVMForScenarioValidateExitCode(ctx, s, steps, 0, "could not validate file has contents - might mean file does not have contents, might mean something went wrong") + } else { + steps := []string{ + "set -ex", + fmt.Sprintf("ls -la %[1]s", fileName), + fmt.Sprintf("sudo cat %[1]s", fileName), + fmt.Sprintf("(sudo cat %[1]s | grep -q -F -e %[2]q)", fileName, contents), + } - command := makeExecutableCommand(steps) - execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not validate file has contents - might mean file does not have contents, might mean something went wrong") + execScriptOnVMForScenarioValidateExitCode(ctx, s, steps, 0, "could not validate file has contents - might mean file does not have contents, might mean something went wrong") + } } func ValidateFileExcludesContent(ctx context.Context, s *Scenario, fileName string, contents string) { require.NotEqual(s.T, "", contents, "Test setup failure: Can't validate that a file excludes an empty string. Filename: %s", fileName) steps := []string{ + "set -ex", fmt.Sprintf("test -f %[1]s || exit 0", fileName), fmt.Sprintf("ls -la %[1]s", fileName), fmt.Sprintf("sudo cat %[1]s", fileName), fmt.Sprintf("(sudo cat %[1]s | grep -q -v -F -e %[2]q)", fileName, contents), } - command := makeExecutableCommand(steps) - execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not validate file excludes contents - might mean file does have contents, might mean something went wrong") -} - -// this function is just used to remove some bash specific tokens so we can echo the command to stdout. -func cleanse(str string) string { - return strings.Replace(str, "'", "", -1) -} - -func makeExecutableCommand(steps []string) string { - stepsWithEchos := make([]string, len(steps)*2) - - for i, s := range steps { - stepsWithEchos[i*2] = fmt.Sprintf("echo '%s'", cleanse(s)) - stepsWithEchos[i*2+1] = s - } - - // quote " quotes and $ vars - joinedCommand := strings.Join(stepsWithEchos, " && ") - quotedCommand := strings.Replace(joinedCommand, "'", "'\"'\"'", -1) - - command := fmt.Sprintf("bash -c '%s'", quotedCommand) - - return command + execScriptOnVMForScenarioValidateExitCode(ctx, s, steps, 0, "could not validate file excludes contents - might mean file does have contents, might mean something went wrong") } func ServiceCanRestartValidator(ctx context.Context, s *Scenario, serviceName string, restartTimeoutInSeconds int) { steps := []string{ + "set -ex", // Verify the service is active - print the state then verify so we have logs fmt.Sprintf("(systemctl -n 5 status %s || true)", serviceName), fmt.Sprintf("systemctl is-active %s", serviceName), @@ -131,8 +139,7 @@ func ServiceCanRestartValidator(ctx context.Context, s *Scenario, serviceName st "if [[ \"$INITIAL_PID\" == \"$POST_PID\" ]]; then echo PID did not change after restart, failing validator. ; exit 1; fi", } - command := makeExecutableCommand(steps) - execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "command to restart service failed") + execScriptOnVMForScenarioValidateExitCode(ctx, s, steps, 0, "command to restart service failed") } func ValidateUlimitSettings(ctx context.Context, s *Scenario, ulimits map[string]string) { @@ -141,8 +148,8 @@ func ValidateUlimitSettings(ctx context.Context, s *Scenario, ulimits map[string ulimitKeys = append(ulimitKeys, k) } - command := fmt.Sprintf("systemctl cat containerd.service | grep -E -i '%s'", strings.Join(ulimitKeys, "|")) - execResult := execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not read containerd.service file") + command := fmt.Sprintf("sudo systemctl cat containerd.service | grep -E -i '%s'", strings.Join(ulimitKeys, "|")) + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{command}, 0, "could not read containerd.service file") for name, value := range ulimits { require.Contains(s.T, execResult.stdout.String(), fmt.Sprintf("%s=%v", name, value), "expected to find %s set to %v, but was not", name, value) @@ -157,14 +164,14 @@ func execOnVMForScenarioOnUnprivilegedPod(ctx context.Context, s *Scenario, cmd return execResult } -func execOnVMForScenario(ctx context.Context, s *Scenario, cmd string) *podExecResult { - result, err := execOnVM(ctx, s.Runtime.Cluster.Kube, s.Runtime.VMPrivateIP, s.Runtime.DebugHostPod, string(s.Runtime.SSHKeyPrivate), cmd) +func execScriptOnVMForScenario(ctx context.Context, s *Scenario, cmd []string) *podExecResult { + result, err := execScriptOnVm(ctx, s, s.Runtime.VMPrivateIP, s.Runtime.DebugHostPod, string(s.Runtime.SSHKeyPrivate), cmd, s.VHD) require.NoError(s.T, err, "failed to execute command on VM") return result } -func execOnVMForScenarioValidateExitCode(ctx context.Context, s *Scenario, cmd string, expectedExitCode int, additionalErrorMessage string) *podExecResult { - execResult := execOnVMForScenario(ctx, s, cmd) +func execScriptOnVMForScenarioValidateExitCode(ctx context.Context, s *Scenario, cmd []string, expectedExitCode int, additionalErrorMessage string) *podExecResult { + execResult := execScriptOnVMForScenario(ctx, s, cmd) expectedExitCodeStr := fmt.Sprint(expectedExitCode) require.Equal(s.T, expectedExitCodeStr, execResult.exitCode, "exec command failed with exit code %q, expected exit code %s\nCommand: %s\nAdditional detail: %s\nSTDOUT:\n%s\n\nSTDERR:\n%s", execResult.exitCode, expectedExitCodeStr, cmd, additionalErrorMessage, execResult.stdout, execResult.stderr) @@ -174,18 +181,18 @@ func execOnVMForScenarioValidateExitCode(ctx context.Context, s *Scenario, cmd s func ValidateInstalledPackageVersion(ctx context.Context, s *Scenario, component, version string) { s.T.Logf("assert %s %s is installed on the VM", component, version) - installedCommand := func() string { + installedCommand := func() []string { switch s.VHD.OS { case config.OSUbuntu: - return "apt list --installed" + return []string{"sudo apt list --installed"} case config.OSMariner, config.OSAzureLinux: - return "dnf list installed" + return []string{"sudo dnf list installed"} default: s.T.Fatalf("command to get package list isn't implemented for OS %s", s.VHD.OS) - return "" + return []string{""} } }() - execResult := execOnVMForScenarioValidateExitCode(ctx, s, installedCommand, 0, "could not get package list") + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, installedCommand, 0, "could not get package list") containsComponent := func() bool { for _, line := range strings.Split(execResult.stdout.String(), "\n") { if strings.Contains(line, component) && strings.Contains(line, version) { @@ -201,30 +208,31 @@ func ValidateInstalledPackageVersion(ctx context.Context, s *Scenario, component } func ValidateKubeletNodeIP(ctx context.Context, s *Scenario) { - execResult := execOnVMForScenarioValidateExitCode(ctx, s, "cat /etc/default/kubelet", 0, "could lot read kubelet config") + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{"sudo cat /etc/default/kubelet"}, 0, "could not read kubelet config") + stdout := execResult.stdout.String() // Search for "--node-ip" flag and its value. - matches := regexp.MustCompile(`--node-ip=([a-zA-Z0-9.,]*)`).FindStringSubmatch(execResult.stdout.String()) - require.NotNil(s.T, matches, "could not find kubelet flag --node-ip") - require.GreaterOrEqual(s.T, len(matches), 2, "could not find kubelet flag --node-ip") + matches := regexp.MustCompile(`--node-ip=([a-zA-Z0-9.,]*)`).FindStringSubmatch(stdout) + require.NotNil(s.T, matches, "could not find kubelet flag --node-ip\nStdout: \n%s", stdout) + require.GreaterOrEqual(s.T, len(matches), 2, "could not find kubelet flag --node-ip.\nStdout: \n%s", stdout) ipAddresses := strings.Split(matches[1], ",") // Could be multiple for dual-stack. - require.GreaterOrEqual(s.T, len(ipAddresses), 1, "expected at least one --node-ip address, but got none") - require.LessOrEqual(s.T, len(ipAddresses), 2, "expected at most two --node-ip addresses, but got %d", len(ipAddresses)) + require.GreaterOrEqual(s.T, len(ipAddresses), 1, "expected at least one --node-ip address, but got none\nStdout: \n%s", stdout) + require.LessOrEqual(s.T, len(ipAddresses), 2, "expected at most two --node-ip addresses, but got %d\nStdout: \n%s", len(ipAddresses), stdout) // Check that each IP is a valid address. for _, ipAddress := range ipAddresses { - require.NotNil(s.T, net.ParseIP(ipAddress), "--node-ip value %q is not a valid IP address", ipAddress) + require.NotNil(s.T, net.ParseIP(ipAddress), "--node-ip value %q is not a valid IP address\nStdout: \n%s", ipAddress, stdout) } } func ValidateIMDSRestrictionRule(ctx context.Context, s *Scenario, table string) { - cmd := fmt.Sprintf("iptables -t %s -S | grep -q 'AKS managed: added by AgentBaker ensureIMDSRestriction for IMDS restriction feature'", table) - execOnVMForScenarioValidateExitCode(ctx, s, cmd, 0, "expected to find IMDS restriction rule, but did not") + cmd := fmt.Sprintf("sudo iptables -t %s -S | grep -q 'AKS managed: added by AgentBaker ensureIMDSRestriction for IMDS restriction feature'", table) + execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{cmd}, 0, "expected to find IMDS restriction rule, but did not") } func ValidateMultipleKubeProxyVersionsExist(ctx context.Context, s *Scenario) { - execResult := execOnVMForScenario(ctx, s, "ctr --namespace k8s.io images list | grep kube-proxy | awk '{print $1}' | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'") + execResult := execScriptOnVMForScenario(ctx, s, []string{"sudo ctr --namespace k8s.io images list | grep kube-proxy | awk '{print $1}' | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'"}) if execResult.exitCode != "0" { s.T.Errorf("Failed to list kube-proxy images: %s", execResult.stderr) return @@ -249,7 +257,7 @@ func ValidateMultipleKubeProxyVersionsExist(ctx context.Context, s *Scenario) { } func ValidateContainerdWASMShims(ctx context.Context, s *Scenario) { - execResult := execOnVMForScenarioValidateExitCode(ctx, s, "cat /etc/containerd/config.toml", 0, "could not get containerd config content") + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{"sudo cat /etc/containerd/config.toml"}, 0, "could not get containerd config content") expectedShims := []string{ `[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin]`, `runtime_type = "io.containerd.spin.v2"`, @@ -281,21 +289,21 @@ func ValidateContainerdWASMShims(ctx context.Context, s *Scenario) { } func ValidateKubeletHasNotStopped(ctx context.Context, s *Scenario) { - command := "journalctl -u kubelet" - execResult := execOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not retrieve kubelet logs") + command := "sudo journalctl -u kubelet" + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{command}, 0, "could not retrieve kubelet logs") assert.NotContains(s.T, execResult.stdout.String(), "Stopped Kubelet") assert.Contains(s.T, execResult.stdout.String(), "Started Kubelet") } func ValidateServicesDoNotRestartKubelet(ctx context.Context, s *Scenario) { // grep all filesin /etc/systemd/system/ for /restart\s+kubelet/ and count results - command := "grep -rl 'restart[[:space:]]\\+kubelet' /etc/systemd/system/" - execOnVMForScenarioValidateExitCode(ctx, s, command, 1, "expected to find no services containing 'restart kubelet' in /etc/systemd/system/") + command := "sudo grep -rl 'restart[[:space:]]\\+kubelet' /etc/systemd/system/" + execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{command}, 1, "expected to find no services containing 'restart kubelet' in /etc/systemd/system/") } // ValidateKubeletHasFlags checks kubelet is started with the right flags and configs. func ValidateKubeletHasFlags(ctx context.Context, s *Scenario, filePath string) { - execResult := execOnVMForScenarioValidateExitCode(ctx, s, `journalctl -u kubelet`, 0, "could not get kubelet logs") + execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, []string{`sudo journalctl -u kubelet`}, 0, "could not get kubelet logs") configFileFlags := fmt.Sprintf("FLAG: --config=\"%s\"", filePath) require.Containsf(s.T, execResult.stdout.String(), configFileFlags, "expected to find flag %s, but not found", "config") } @@ -365,3 +373,18 @@ func ValidateRunc12Properties(ctx context.Context, s *Scenario, versions []strin require.Truef(s.T, strings.HasPrefix(versions[0], "1.2."), "expected moby-runc version to start with '1.2.', got %v", versions[0]) ValidateInstalledPackageVersion(ctx, s, "moby-runc", versions[0]) } + +func ValidateWindowsProcessHasCliArguments(ctx context.Context, s *Scenario, processName string, arguments []string) { + steps := []string{ + fmt.Sprintf("(Get-CimInstance Win32_Process -Filter \"name='%[1]s'\")[0].CommandLine", processName), + } + + podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, steps, 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong") + + actualArgs := strings.Split(podExecResult.stdout.String(), " ") + + for i := 0; i < len(arguments); i++ { + expectedArgument := arguments[i] + require.Contains(s.T, actualArgs, expectedArgument) + } +} diff --git a/e2e/vmss.go b/e2e/vmss.go index 0ea5bf912aa..aad32a58d27 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -115,11 +115,11 @@ func extractLogsFromVMLinux(ctx context.Context, s *Scenario) { require.NoError(s.T, err) commandList := map[string]string{ - "cluster-provision.log": "cat /var/log/azure/cluster-provision.log", - "kubelet.log": "journalctl -u kubelet", - "cluster-provision-cse-output.log": "cat /var/log/azure/cluster-provision-cse-output.log", - "sysctl-out.log": "sysctl -a", - "aks-node-controller.log": "cat /var/log/azure/aks-node-controller.log", + "cluster-provision.log": "sudo cat /var/log/azure/cluster-provision.log", + "kubelet.log": "sudo journalctl -u kubelet", + "cluster-provision-cse-output.log": "sudo cat /var/log/azure/cluster-provision-cse-output.log", + "sysctl-out.log": "sudo sysctl -a", + "aks-node-controller.log": "sudo cat /var/log/azure/aks-node-controller.log", } pod, err := s.Runtime.Cluster.Kube.GetHostNetworkDebugPod(ctx, s.T) @@ -129,7 +129,7 @@ func extractLogsFromVMLinux(ctx context.Context, s *Scenario) { var logFiles = map[string]string{} for file, sourceCmd := range commandList { - execResult, err := execOnVM(ctx, s.Runtime.Cluster.Kube, privateIP, pod.Name, string(s.Runtime.SSHKeyPrivate), sourceCmd) + execResult, err := execBashCommandOnVM(ctx, s, privateIP, pod.Name, string(s.Runtime.SSHKeyPrivate), sourceCmd) if err != nil { s.T.Logf("error executing %s: %s", sourceCmd, err) continue