From db712e7eed983ea38ad1bed369b36b71dfc70c64 Mon Sep 17 00:00:00 2001 From: Anton Troshin Date: Tue, 1 Oct 2024 11:19:35 -0500 Subject: [PATCH] Fixing e2e tests, podman and scheduler fail with mariner images (#1450) * Fix standalone e2e tests Fix podman e2e install Fix scheduler start failure on standalone mariner image variant Signed-off-by: Anton Troshin * fix test Signed-off-by: Anton Troshin * revert Signed-off-by: Anton Troshin * Fix podman machine settings Add cleanups Remove parallel tests Fix mariner volume mount location Remove old build tags Signed-off-by: Anton Troshin --------- Signed-off-by: Anton Troshin --- .github/workflows/kind_e2e.yaml | 3 +- .github/workflows/self_hosted_e2e.yaml | 5 ++- pkg/standalone/standalone.go | 12 +++++- tests/e2e/standalone/commands.go | 1 - tests/e2e/standalone/init_negative_test.go | 5 --- .../standalone/init_run_custom_path_test.go | 1 - tests/e2e/standalone/init_test.go | 16 ++++---- tests/e2e/standalone/invoke_test.go | 1 - tests/e2e/standalone/list_test.go | 8 +++- tests/e2e/standalone/publish_test.go | 1 - tests/e2e/standalone/run_template_test.go | 26 ++++++------- tests/e2e/standalone/run_test.go | 1 - tests/e2e/standalone/stop_test.go | 10 ++++- .../standalone/stop_with_run_template_test.go | 21 +++++++--- tests/e2e/standalone/uninstall_test.go | 1 - tests/e2e/standalone/utils.go | 38 ++++++++++++++++++- tests/e2e/standalone/version_test.go | 1 - .../standalone/windows_run_template_test.go | 2 - 18 files changed, 102 insertions(+), 51 deletions(-) diff --git a/.github/workflows/kind_e2e.yaml b/.github/workflows/kind_e2e.yaml index af559b29a..ca5a110e2 100644 --- a/.github/workflows/kind_e2e.yaml +++ b/.github/workflows/kind_e2e.yaml @@ -50,11 +50,10 @@ jobs: name: E2E tests for K8s (KinD) runs-on: ubuntu-latest env: - DAPR_RUNTIME_PINNED_VERSION: 1.13.0-rc.2 + DAPR_RUNTIME_PINNED_VERSION: 1.13.5 DAPR_DASHBOARD_PINNED_VERSION: 0.14.0 DAPR_RUNTIME_LATEST_STABLE_VERSION: DAPR_DASHBOARD_LATEST_STABLE_VERSION: - DAPR_TGZ: dapr-1.13.0-rc.2.tgz strategy: fail-fast: false # Keep running if one leg fails. matrix: diff --git a/.github/workflows/self_hosted_e2e.yaml b/.github/workflows/self_hosted_e2e.yaml index c8c0441ba..3e67f4cef 100644 --- a/.github/workflows/self_hosted_e2e.yaml +++ b/.github/workflows/self_hosted_e2e.yaml @@ -38,12 +38,12 @@ jobs: GOARCH: ${{ matrix.target_arch }} GOPROXY: https://proxy.golang.org ARCHIVE_OUTDIR: dist/archives - DAPR_RUNTIME_PINNED_VERSION: "1.13.0-rc.2" + DAPR_RUNTIME_PINNED_VERSION: "1.13.5" DAPR_DASHBOARD_PINNED_VERSION: 0.14.0 DAPR_RUNTIME_LATEST_STABLE_VERSION: "" DAPR_DASHBOARD_LATEST_STABLE_VERSION: "" GOLANG_PROTOBUF_REGISTRATION_CONFLICT: warn - PODMAN_VERSION: 4.4.4 + PODMAN_VERSION: 4.9.3 strategy: # TODO: Remove this when our E2E tests are stable for podman on MacOS. fail-fast: false # Keep running if one leg fails. @@ -122,6 +122,7 @@ jobs: sudo podman-mac-helper install podman machine init podman machine start --log-level debug + podman machine ssh sudo sysctl -w kernel.keys.maxkeys=20000 echo "CONTAINER_RUNTIME=podman" >> $GITHUB_ENV - name: Determine latest Dapr Runtime version including Pre-releases if: github.base_ref == 'master' diff --git a/pkg/standalone/standalone.go b/pkg/standalone/standalone.go index bc7fba008..8dcca112d 100644 --- a/pkg/standalone/standalone.go +++ b/pkg/standalone/standalone.go @@ -652,7 +652,11 @@ func runSchedulerService(wg *sync.WaitGroup, errorChan chan<- error, info initIn // /var/lock and can therefore mount the Docker volume here. // TODO: update the Dapr scheduler dockerfile to create a scheduler user id writeable // directory at /var/lib/dapr/scheduler, then update the path here. - args = append(args, "--volume", *info.schedulerVolume+":/var/lock") + if strings.EqualFold(info.imageVariant, "mariner") { + args = append(args, "--volume", *info.schedulerVolume+":/var/tmp") + } else { + args = append(args, "--volume", *info.schedulerVolume+":/var/lock") + } } if info.dockerNetwork != "" { @@ -673,7 +677,11 @@ func runSchedulerService(wg *sync.WaitGroup, errorChan chan<- error, info initIn ) } - args = append(args, image, "--etcd-data-dir=/var/lock/dapr/scheduler") + if strings.EqualFold(info.imageVariant, "mariner") { + args = append(args, image, "--etcd-data-dir=/var/tmp/dapr/scheduler") + } else { + args = append(args, image, "--etcd-data-dir=/var/lock/dapr/scheduler") + } _, err = utils.RunCmdAndWait(runtimeCmd, args...) if err != nil { diff --git a/tests/e2e/standalone/commands.go b/tests/e2e/standalone/commands.go index 7e35f34e8..efa1de751 100644 --- a/tests/e2e/standalone/commands.go +++ b/tests/e2e/standalone/commands.go @@ -1,5 +1,4 @@ //go:build e2e || template -// +build e2e template /* Copyright 2022 The Dapr Authors diff --git a/tests/e2e/standalone/init_negative_test.go b/tests/e2e/standalone/init_negative_test.go index 9fe67d5ee..adfa6549d 100644 --- a/tests/e2e/standalone/init_negative_test.go +++ b/tests/e2e/standalone/init_negative_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors @@ -33,7 +32,6 @@ func TestStandaloneInitNegatives(t *testing.T) { require.NoError(t, err, "expected no error on querying for os home dir") t.Run("run without install", func(t *testing.T) { - t.Parallel() output, err := cmdRun("") require.Error(t, err, "expected error status on run without install") path := filepath.Join(homeDir, ".dapr", "components") @@ -45,21 +43,18 @@ func TestStandaloneInitNegatives(t *testing.T) { }) t.Run("list without install", func(t *testing.T) { - t.Parallel() output, err := cmdList("") require.NoError(t, err, "expected no error status on list without install") require.Equal(t, "No Dapr instances found.\n", output) }) t.Run("stop without install", func(t *testing.T) { - t.Parallel() output, err := cmdStopWithAppID("test") require.NoError(t, err, "expected no error on stop without install") require.Contains(t, output, "failed to stop app id test: couldn't find app id test", "expected output to match") }) t.Run("uninstall without install", func(t *testing.T) { - t.Parallel() output, err := cmdUninstall() require.NoError(t, err, "expected no error on uninstall without install") require.Contains(t, output, "Removing Dapr from your machine...", "expected output to contain message") diff --git a/tests/e2e/standalone/init_run_custom_path_test.go b/tests/e2e/standalone/init_run_custom_path_test.go index 2b684d01b..da7550a89 100644 --- a/tests/e2e/standalone/init_run_custom_path_test.go +++ b/tests/e2e/standalone/init_run_custom_path_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors diff --git a/tests/e2e/standalone/init_test.go b/tests/e2e/standalone/init_test.go index e07b405a6..3497cc98c 100644 --- a/tests/e2e/standalone/init_test.go +++ b/tests/e2e/standalone/init_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors @@ -167,15 +166,17 @@ func TestStandaloneInit(t *testing.T) { placementPort = 6050 } - verifyTCPLocalhost(t, placementPort) + verifyTCPLocalhost(t, placementPort) }) t.Run("init version with scheduler", func(t *testing.T) { // Ensure a clean environment must(t, cmdUninstall, "failed to uninstall Dapr") + latestDaprRuntimeVersion, latestDaprDashboardVersion := common.GetVersionsFromEnv(t, true) + args := []string{ - "--runtime-version", "1.14.0-rc.3", + "--runtime-version", latestDaprRuntimeVersion, "--dev", } output, err := cmdInit(args...) @@ -189,9 +190,8 @@ func TestStandaloneInit(t *testing.T) { daprPath := filepath.Join(homeDir, ".dapr") require.DirExists(t, daprPath, "Directory %s does not exist", daprPath) - _, latestDaprDashboardVersion := common.GetVersionsFromEnv(t, true) - verifyContainers(t, "1.14.0-rc.3") - verifyBinaries(t, daprPath, "1.14.0-rc.3", latestDaprDashboardVersion) + verifyContainers(t, latestDaprRuntimeVersion) + verifyBinaries(t, daprPath, latestDaprRuntimeVersion, latestDaprDashboardVersion) verifyConfigs(t, daprPath) placementPort := 50005 @@ -201,8 +201,8 @@ func TestStandaloneInit(t *testing.T) { schedulerPort = 6060 } - verifyTCPLocalhost(t, placementPort) - verifyTCPLocalhost(t, schedulerPort) + verifyTCPLocalhost(t, placementPort) + verifyTCPLocalhost(t, schedulerPort) }) t.Run("init without runtime-version flag with mariner images", func(t *testing.T) { diff --git a/tests/e2e/standalone/invoke_test.go b/tests/e2e/standalone/invoke_test.go index 812d0ea4b..c21ec212b 100644 --- a/tests/e2e/standalone/invoke_test.go +++ b/tests/e2e/standalone/invoke_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors diff --git a/tests/e2e/standalone/list_test.go b/tests/e2e/standalone/list_test.go index a33345644..527642163 100644 --- a/tests/e2e/standalone/list_test.go +++ b/tests/e2e/standalone/list_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors @@ -137,6 +136,13 @@ func TestStandaloneList(t *testing.T) { t.Log(output) require.NoError(t, err, "expected no error status on list") require.Equal(t, "No Dapr instances found.\n", output) + + // This test is skipped on Windows, but just in case, try to terminate dashboard process only on non-Windows OSs + // stopProcess uses kill command, won't work on windows + if runtime.GOOS != "windows" { + err = stopProcess("dashboard", "--port", "5555") + require.NoError(t, err, "failed to stop dashboard process") + } }) } diff --git a/tests/e2e/standalone/publish_test.go b/tests/e2e/standalone/publish_test.go index e7e334a75..845eb5576 100644 --- a/tests/e2e/standalone/publish_test.go +++ b/tests/e2e/standalone/publish_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors diff --git a/tests/e2e/standalone/run_template_test.go b/tests/e2e/standalone/run_template_test.go index 4ef006b61..907fd8fc0 100644 --- a/tests/e2e/standalone/run_template_test.go +++ b/tests/e2e/standalone/run_template_test.go @@ -1,6 +1,4 @@ //go:build !windows && (e2e || template) -// +build !windows -// +build e2e template /* Copyright 2023 The Dapr Authors @@ -23,7 +21,6 @@ import ( "context" "fmt" "io/ioutil" - "os" "path/filepath" "strings" "testing" @@ -43,6 +40,7 @@ type AppTestOutput struct { } func TestRunWithTemplateFile(t *testing.T) { + cleanUpLogs() ensureDaprInstallation(t) t.Cleanup(func() { // remove dapr installation after all tests in this function. @@ -54,8 +52,7 @@ func TestRunWithTemplateFile(t *testing.T) { runFilePath := "../testdata/run-template-files/wrong_emit_metrics_app_dapr.yaml" t.Cleanup(func() { // assumption in the test is that there is only one set of app and daprd logs in the logs directory. - os.RemoveAll("../../apps/emit-metrics/.dapr/logs") - os.RemoveAll("../../apps/processor/.dapr/logs") + cleanUpLogs() stopAllApps(t, runFilePath) }) args := []string{ @@ -104,8 +101,7 @@ func TestRunWithTemplateFile(t *testing.T) { runFilePath := "../testdata/run-template-files/dapr.yaml" t.Cleanup(func() { // assumption in the test is that there is only one set of app and daprd logs in the logs directory. - os.RemoveAll("../../apps/emit-metrics/.dapr/logs") - os.RemoveAll("../../apps/processor/.dapr/logs") + cleanUpLogs() stopAllApps(t, runFilePath) }) args := []string{ @@ -161,8 +157,7 @@ func TestRunWithTemplateFile(t *testing.T) { runFilePath := "../testdata/run-template-files/env_var_not_set_dapr.yaml" t.Cleanup(func() { // assumption in the test is that there is only one set of app and daprd logs in the logs directory. - os.RemoveAll("../../apps/emit-metrics/.dapr/logs") - os.RemoveAll("../../apps/processor/.dapr/logs") + cleanUpLogs() stopAllApps(t, runFilePath) }) args := []string{ @@ -212,8 +207,7 @@ func TestRunWithTemplateFile(t *testing.T) { runFilePath := "../testdata/run-template-files/no_app_command.yaml" t.Cleanup(func() { // assumption in the test is that there is only one set of app and daprd logs in the logs directory. - os.RemoveAll("../../apps/emit-metrics/.dapr/logs") - os.RemoveAll("../../apps/processor/.dapr/logs") + cleanUpLogs() stopAllApps(t, runFilePath) }) args := []string{ @@ -264,8 +258,7 @@ func TestRunWithTemplateFile(t *testing.T) { runFilePath := "../testdata/run-template-files/empty_app_command.yaml" t.Cleanup(func() { // assumption in the test is that there is only one set of app and daprd logs in the logs directory. - os.RemoveAll("../../apps/emit-metrics/.dapr/logs") - os.RemoveAll("../../apps/processor/.dapr/logs") + cleanUpLogs() stopAllApps(t, runFilePath) }) args := []string{ @@ -313,8 +306,7 @@ func TestRunWithTemplateFile(t *testing.T) { runFilePath := "../testdata/run-template-files/app_output_to_file_and_console.yaml" t.Cleanup(func() { // assumption in the test is that there is only one set of app and daprd logs in the logs directory. - os.RemoveAll("../../apps/emit-metrics/.dapr/logs") - os.RemoveAll("../../apps/processor/.dapr/logs") + cleanUpLogs() stopAllApps(t, runFilePath) }) args := []string{ @@ -372,6 +364,10 @@ func TestRunTemplateFileWithoutDaprInit(t *testing.T) { // remove any dapr installation before this test. must(t, cmdUninstall, "failed to uninstall Dapr") t.Run("valid template file without dapr init", func(t *testing.T) { + t.Cleanup(func() { + // assumption in the test is that there is only one set of app and daprd logs in the logs directory. + cleanUpLogs() + }) args := []string{ "-f", "../testdata/run-template-files/no_app_command.yaml", } diff --git a/tests/e2e/standalone/run_test.go b/tests/e2e/standalone/run_test.go index 934f4fd0c..7e95696e8 100644 --- a/tests/e2e/standalone/run_test.go +++ b/tests/e2e/standalone/run_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors diff --git a/tests/e2e/standalone/stop_test.go b/tests/e2e/standalone/stop_test.go index c14ac364a..f0c25b30b 100644 --- a/tests/e2e/standalone/stop_test.go +++ b/tests/e2e/standalone/stop_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors @@ -18,6 +17,7 @@ package standalone_test import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,6 +25,14 @@ import ( func TestStandaloneStop(t *testing.T) { ensureDaprInstallation(t) + + time.Sleep(5 * time.Second) + + t.Cleanup(func() { + // remove dapr installation after all tests in this function. + must(t, cmdUninstall, "failed to uninstall Dapr") + }) + executeAgainstRunningDapr(t, func() { t.Run("stop", func(t *testing.T) { output, err := cmdStopWithAppID("dapr_e2e_stop") diff --git a/tests/e2e/standalone/stop_with_run_template_test.go b/tests/e2e/standalone/stop_with_run_template_test.go index ae4adaf14..ce400284f 100644 --- a/tests/e2e/standalone/stop_with_run_template_test.go +++ b/tests/e2e/standalone/stop_with_run_template_test.go @@ -1,6 +1,4 @@ //go:build !windows && (e2e || template) -// +build !windows -// +build e2e template /* Copyright 2023 The Dapr Authors @@ -22,7 +20,6 @@ package standalone_test import ( "encoding/json" "fmt" - "os" "testing" "time" @@ -31,6 +28,9 @@ import ( ) func TestStopAppsStartedWithRunTemplate(t *testing.T) { + // clean up logs before starting the tests + cleanUpLogs() + ensureDaprInstallation(t) t.Cleanup(func() { // remove dapr installation after all tests in this function. @@ -38,6 +38,9 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) { }) t.Run("stop apps by passing run template file", func(t *testing.T) { + t.Cleanup(func() { + cleanUpLogs() + }) go ensureAllAppsStartedWithRunTemplate(t) time.Sleep(10 * time.Second) cliPID := getCLIPID(t) @@ -50,6 +53,9 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) { }) t.Run("stop apps by passing a directory containing dapr.yaml", func(t *testing.T) { + t.Cleanup(func() { + cleanUpLogs() + }) go ensureAllAppsStartedWithRunTemplate(t) time.Sleep(10 * time.Second) cliPID := getCLIPID(t) @@ -60,6 +66,9 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) { }) t.Run("stop apps by passing an invalid directory", func(t *testing.T) { + t.Cleanup(func() { + cleanUpLogs() + }) go ensureAllAppsStartedWithRunTemplate(t) time.Sleep(10 * time.Second) output, err := cmdStopWithRunTemplate("../testdata/invalid-dir") @@ -72,6 +81,9 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) { }) t.Run("stop apps started with run template", func(t *testing.T) { + t.Cleanup(func() { + cleanUpLogs() + }) go ensureAllAppsStartedWithRunTemplate(t) time.Sleep(10 * time.Second) cliPID := getCLIPID(t) @@ -95,8 +107,7 @@ func ensureAllAppsStartedWithRunTemplate(t *testing.T) { func tearDownTestSetup(t *testing.T) { // remove dapr installation after all tests in this function. must(t, cmdUninstall, "failed to uninstall Dapr") - os.RemoveAll("../../apps/emit-metrics/.dapr/logs") - os.RemoveAll("../../apps/processor/.dapr/logs") + cleanUpLogs() } func getCLIPID(t *testing.T) string { diff --git a/tests/e2e/standalone/uninstall_test.go b/tests/e2e/standalone/uninstall_test.go index ae1545a99..5d9cf65e3 100644 --- a/tests/e2e/standalone/uninstall_test.go +++ b/tests/e2e/standalone/uninstall_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors diff --git a/tests/e2e/standalone/utils.go b/tests/e2e/standalone/utils.go index 87d2fa78c..b41b50754 100644 --- a/tests/e2e/standalone/utils.go +++ b/tests/e2e/standalone/utils.go @@ -1,5 +1,4 @@ //go:build e2e || template -// +build e2e template /* Copyright 2022 The Dapr Authors @@ -157,3 +156,40 @@ func containerRuntime() string { } return "" } + +func getRunningProcesses() []string { + cmd := exec.Command("ps", "-o", "pid,command") + output, err := cmd.Output() + if err != nil { + return nil + } + + processes := strings.Split(string(output), "\n") + + // clean the process output whitespace + for i, process := range processes { + processes[i] = strings.TrimSpace(process) + } + return processes +} + +func stopProcess(args ...string) error { + processCommand := strings.Join(args, " ") + processes := getRunningProcesses() + for _, process := range processes { + if strings.Contains(process, processCommand) { + processSplit := strings.SplitN(process, " ", 2) + cmd := exec.Command("kill", "-9", processSplit[0]) + err := cmd.Run() + if err != nil { + return err + } + } + } + return nil +} + +func cleanUpLogs() { + os.RemoveAll("../../apps/emit-metrics/.dapr/logs") + os.RemoveAll("../../apps/processor/.dapr/logs") +} diff --git a/tests/e2e/standalone/version_test.go b/tests/e2e/standalone/version_test.go index 269389998..c3372a168 100644 --- a/tests/e2e/standalone/version_test.go +++ b/tests/e2e/standalone/version_test.go @@ -1,5 +1,4 @@ //go:build e2e && !template -// +build e2e,!template /* Copyright 2022 The Dapr Authors diff --git a/tests/e2e/standalone/windows_run_template_test.go b/tests/e2e/standalone/windows_run_template_test.go index c570f94c7..b53f64a3b 100644 --- a/tests/e2e/standalone/windows_run_template_test.go +++ b/tests/e2e/standalone/windows_run_template_test.go @@ -1,6 +1,4 @@ //go:build windows && (e2e || template) -// +build windows -// +build e2e template /* Copyright 2023 The Dapr Authors