diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1a2141b398..d0601c9c1b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,4 +1,4 @@ -name: E2E Tests +name: E2E Tests: Host-based on: [pull_request] @@ -9,17 +9,21 @@ jobs: os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} steps: + - name: Set Environment Variables + run: | + echo "KUBECONFIG=${{ github.workspace }}/hack/bin/kubeconfig.yaml" >> "$GITHUB_ENV" + echo "PATH=${{ github.workspace }}/hack/bin:$PATH" >> "$GITHUB_ENV" - uses: actions/checkout@v3 - uses: ./.github/composite/go-setup - name: Install Binaries - run: ./hack/binaries.sh + run: ./hack/install-binaries.sh - name: Allocate Cluster run: ./hack/allocate.sh - name: Local Registry run: ./hack/registry.sh - name: E2E Test - run: make test-e2e + run: make test-e2e-host - uses: codecov/codecov-action@v3 with: files: ./coverage.txt - flags: e2e-test + flags: e2e diff --git a/Makefile b/Makefile index 0da5d15f3e..24ac71f2bb 100644 --- a/Makefile +++ b/Makefile @@ -200,9 +200,7 @@ test-integration: ## Run integration tests using an available cluster. go test -ldflags "$(LDFLAGS)" -tags integration -timeout 30m --coverprofile=coverage.txt ./... -v -test-e2e-quick: func-instrumented ## Run quick end-to-end tests using an available cluster. - # go test ./e2e -tags="e2e" - go test ./e2e +.PHONY: func-instrumented func-instrumented: ## Func binary that is instrumented for e2e tests env CGO_ENABLED=1 go build -ldflags "$(LDFLAGS)" -cover -o func ./cmd/$(BIN) @@ -216,6 +214,9 @@ test-e2e-runtime: func-instrumented ## Run end-to-end lifecycle tests using an a test-e2e-on-cluster: func-instrumented ## Run end-to-end on-cluster build tests using an available cluster. ./test/e2e_oncluster_tests.sh +test-e2e-host: func-instrumented ## Run end-to-end tests for the host builder + go test ./e2e -tags="e2e" + ###################### ##@ Release Artifacts ###################### diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index b90b62880f..9b20ac47f1 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -1,7 +1,15 @@ /* Package e2e provides an end-to-end test suite for the Functions CLI "func". -Status: +tl;dr: + + ./hack/registry.sh Configure system for insecure local registrires + ./hack/install-binaries.sh Fetch binaries into ./hack/bin + ./hack/allocate.sh Create a cluster and kube config in ./hack/bin + go test ./e2e -tags=e2e Run all tests using these bins and cluster + ./hack/delete.sh Nuke the cluster + +Test Suite Status: This package is a work-in-progress, and is not being executed in CI. For the active e2e tests, see the "test" directory. @@ -98,6 +106,13 @@ outside of a container (host builder, or runner with --container=false). This can be used to test against specific go versions. Defaults to the go binary found in the current session's PATH. +FUNC_E2E_GIT: the path to the 'git' binary tests should provide to the commands +being tested for git-related activities. Defaults to the git binary +found in the current session's PATH. + +FUNC_E2E_VERBOSE: instructs the test suite to run all commands in +verbose mode. + Running: From the root of the repository, run "make test-e2e-quick". This will compile @@ -117,23 +132,29 @@ the cluster between full test runs. To remove the local cluster, use the "delete.sh" script described above. Upgrades: -- Now supports testing func when a plugin of a different name -- Now supports running specific runtimes rathern than the prior version which supported one or all. -- Uses sensible defaults for environment variables to reduce setup when running locally. -- Removes redundant `go test` flags -- Now supports specifying builders -- Subsets of test can be specified using name prefixes --run=TestCore etc. + - Now supports testing func when a plugin of a different name + - Now supports running specific runtimes rather than the prior version which + supported one or all. + - Uses sensible defaults for environment variables to reduce setup when + running locally. + - Removes redundant `go test` flags + - Now supports specifying builders + - Subsets of test can be specified using name prefixes --run=TestCore etc. */ package e2e import ( "fmt" + "io" "net/http" + "net/http/httputil" "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" + "time" fn "knative.dev/func/pkg/functions" ) @@ -168,25 +189,25 @@ const ( // overridden using FUNC_E2E_KUBECONFIG. DefaultKubeconfig = "../hack/bin/kubeconfig.yaml" - // DefaultName for Functions created when no special name is necessary - // to set up test conditions. - DefaultName = "testfunc" - // DefaultRegistry to use when running the e2e tests. This is the URL // of the registry created by default when using the allocate.sh script // to set up a local testing cluster, but can be customized with // FUNC_E2E_REGISTRY. DefaultRegistry = "localhost:50000/func" + + // DefaultVerbose sets the default for the --verbose flag of all commands. + DefaultVerbose = false ) var ( // static-ish + // DefaultBuilders which we want THESE e2e tests to consider. // This is currently equivalent to all known builders; host, s2i and pack. // Note this only affects tests which are explicitly intended to check // runtimes and builder compatibility. Core tests all use the Go+host builder // combination. - DefaultBuilders = []string{"host", "pack", "s2i"} + // DefaultRuntimes which we want THESE e2e tests to consider // This is currently a subset but will be expanded to be all core runtimes // as they become supported by the Go builder. @@ -245,15 +266,25 @@ var ( // FUNC_E2E_GO. Go string + // Git is the path to the git binary to be provided to commands to use + // which utilize git features. For example when building containers, + // the current git version is provided to the running function as an + // environment variable. This will default to the git found in PATH, but + // can be overridden with FUNC_E2E_GIT. + Git string + // Home is the final path to the default Home directory used for tests // which do not set it explicitly. Home string + + // Verbose mode for all command runs. + Verbose bool ) // --------------------------------------------------------------------------- // CORE TESTS // Create, Read, Update Delete and List. -// Implemented as "init", "run", "deploy", "describe", "list" and "delete" +// Implemented as "create", "run", "deploy", "describe", "list" and "delete" // --------------------------------------------------------------------------- // TestCore_init ensures that initializing a default Function with only the @@ -262,12 +293,9 @@ var ( func TestCore_init(t *testing.T) { // Assemble resetEnv() - root := cdTemp(t) - for _, env := range os.Environ() { - t.Log(env) - } + root := cdTemp(t, "create") - // Act + // Act (newCmd == "func ...") if err := newCmd(t, "init", "-l=go").Run(); err != nil { t.Fatal(err) } @@ -275,10 +303,10 @@ func TestCore_init(t *testing.T) { // Assert f, err := fn.NewFunction(root) if err != nil { - t.Fatal(err) + t.Fatalf("expected an initialized function, but when reading it, got error. %v", err) } if f.Runtime != "go" { - t.Fatalf("expected runtime 'go' got '%v'", f.Runtime) + t.Fatalf("expected initialized function with runtime 'go' got '%v'", f.Runtime) } } @@ -286,62 +314,132 @@ func TestCore_init(t *testing.T) { // becoming available and will echo requests. func TestCore_run(t *testing.T) { resetEnv() - _ = cdTemp(t) + _ = cdTemp(t, "run") // sets Function name obliquely, see docs if err := newCmd(t, "init", "-l=go").Run(); err != nil { t.Fatal(err) } - cmd := newCmd(t, "run", "--container=false") + + cmd := newCmd(t, "run") if err := cmd.Start(); err != nil { t.Fatal(err) } - // TODO: parse output and find final port in the case of a port collision - // the system will use successively higher ports until it finds one - // unoccupied. + if !waitFor(t, "http://localhost:8080") { + t.Fatalf("service does not appear to have started correctly.") + } + + // ^C the running function + if err := cmd.Process.Signal(os.Interrupt); err != nil { + fmt.Fprintf(os.Stderr, "error interrupting. %v", err) + } - res, err := http.Get("localhost:8080") - if err != nil { + // Wait for exit and error if anything other than 130 (^C/interrupt) + if err := cmd.Wait(); isAbnormalExit(t, err) { + t.Fatalf("funciton exited abnormally %v", err) + } +} + +// TestCore_deploy ensures that a function can be deployed to the cluster. +func TestCore_deploy(t *testing.T) { + resetEnv() + _ = cdTemp(t, "deploy") // sets Function name obliquely, see function docs + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { t.Fatal(err) } - cmd.Wait() + cmd := newCmd(t, "deploy") - // note that --container=false will become the default once the scaffolding - // and host builder is supported by most/all core languages. + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + if err := newCmd(t, "delete").Run(); err != nil { + t.Logf("Error deleting function. %v", err) + } + }() + if err := cmd.Wait(); err != nil { + t.Fatalf("deploy error. %v", err) + } + + if !waitFor(t, "http://deploy.default.127.0.0.1.sslip.io") { + t.Fatalf("function did not deploy correctly") + } } -// Removed -// Tests removed from E2Es that need to have an equivalent unit or integration -// test implemented: -// - Interactive Terminal (prompt) tests +// TestCore_update ensures that a running funciton can be updated. +func TestCore_update(t *testing.T) { + resetEnv() + root := cdTemp(t, "update") // sets Function name obliquely, see function docs + + // create + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + + // deploy + if err := newCmd(t, "deploy").Run(); err != nil { + t.Fatal(err) + } + defer func() { + if err := newCmd(t, "delete").Run(); err != nil { + t.Logf("Error deleting function. %v", err) + } + }() + if !waitFor(t, "http://update.default.127.0.0.1.sslip.io") { + t.Fatalf("function did not deploy correctly") + } -// TODO: -// Add "run" both containerised and not for all languages as it is -// implemented. + // update + update := ` + package function + import "fmt" + import "net/http" + func Handle(w http.ResponseWriter, _ *http.Request) { + fmt.Fprintln(w, "UPDATED") + } + ` + err := os.WriteFile(filepath.Join(root, "handle.go"), []byte(update), 0644) + if err != nil { + t.Fatal(err) + } + if err := newCmd(t, "deploy").Run(); err != nil { + t.Fatal(err) + } + + // TODO: change to wait for echo of something in particular that + // ensures the above update took. + if !waitForContent(t, "http://update.default.127.0.0.1.sslip.io", "UPDATED") { + t.Fatalf("function did not update correctly") + } +} // ---------------------------------------------------------------------------- -// Helpers +// Initialization // ---------------------------------------------------------------------------- +// Deprecated Available Settings Final +// --------------------------------------------------- +// E2E_FUNC_BIN => FUNC_E2E_BIN => Bin +// E2E_USE_KN_FUNC => FUNC_E2E_PLUGIN => Plugin +// E2E_REGISTRY_URL => FUNC_E2E_REGISTRY => Registry +// E2E_RUNTIMES => FUNC_E2E_RUNTIMES => Runtimes +// FUNC_E2E_BUILDERS => Builders +// FUNC_E2E_KUBECONFIG => Kubeconfig +// FUNC_E2E_GOCOVERDIR => Gocoverdir +// FUNC_E2E_GO => Go +// FUNC_E2E_GIT => Git // init global settings for the current run from environment -// we readd E2E config settings passed via the FUNC_E2E_* environment +// we read E2E config settings passed via the FUNC_E2E_* environment // variables. These globals are used when creating test cases. // Some tests pass these values as flags, sometimes as environment variables, // sometimes not at all; hence why the actual environment setup is deferred // into each test, merely reading them in here during E2E process init. func init() { fmt.Fprintln(os.Stderr, "Initializing E2E Tests") - // Deprecated Available Settings Final - // --------------------------------------------------- - // E2E_FUNC_BIN => FUNC_E2E_BIN => Bin - // E2E_USE_KN_FUNC => FUNC_E2E_PLUGIN => Plugin - // E2E_REGISTRY_URL => FUNC_E2E_REGISTRY => Registry - // E2E_RUNTIMES => FUNC_E2E_RUNTIMES => Runtimes - // FUNC_E2E_BUILDERS => Builders - // FUNC_E2E_KUBECONFIG => Kubeconfig - // FUNC_E2E_GOCOVERDIR => Gocoverdir + fmt.Fprintln(os.Stderr, "----------------------") fmt.Fprintln(os.Stderr, "Config Provided:") fmt.Fprintf(os.Stderr, " FUNC_E2E_BIN=%v\n", os.Getenv("FUNC_E2E_BIN")) @@ -351,156 +449,159 @@ func init() { fmt.Fprintf(os.Stderr, " FUNC_E2E_PLUGIN=%v\n", os.Getenv("FUNC_E2E_PLUGIN")) fmt.Fprintf(os.Stderr, " FUNC_E2E_REGISTRY=%v\n", os.Getenv("FUNC_E2E_REGISTRY")) fmt.Fprintf(os.Stderr, " FUNC_E2E_RUNTIMES=%v\n", os.Getenv("FUNC_E2E_RUNTIMES")) + fmt.Fprintf(os.Stderr, " FUNC_E2E_GO=%v\n", os.Getenv("FUNC_E2E_GO")) + fmt.Fprintf(os.Stderr, " FUNC_E2E_GIT=%v\n", os.Getenv("FUNC_E2E_GIT")) + fmt.Fprintf(os.Stderr, " FUNC_E2E_VERBOSE=%v\n", os.Getenv("FUNC_E2E_VERBOSE")) fmt.Fprintf(os.Stderr, " (deprecated) E2E_FUNC_BIN=%v\n", os.Getenv("E2E_FUNC_BIN")) fmt.Fprintf(os.Stderr, " (deprecated) E2E_REGISTRY_URL=%v\n", os.Getenv("E2E_REGISTRY_URL")) fmt.Fprintf(os.Stderr, " (deprecated) E2E_RUNTIMES=%v\n", os.Getenv("E2E_RUNTIMES")) fmt.Fprintf(os.Stderr, " (deprecated) E2E_USE_KN_FUNC=%v\n", os.Getenv("E2E_USE_KN_FUNC")) fmt.Fprintln(os.Stderr, "---------------------") + readEnvs() + fmt.Fprintln(os.Stderr, "Final Config:") + fmt.Fprintf(os.Stderr, " Bin=%v\n", Bin) + fmt.Fprintf(os.Stderr, " Plugin=%v\n", Plugin) + fmt.Fprintf(os.Stderr, " Registry=%v\n", Registry) + fmt.Fprintf(os.Stderr, " Runtimes=%v\n", toCSV(Runtimes)) + fmt.Fprintf(os.Stderr, " Builders=%v\n", toCSV(Builders)) + fmt.Fprintf(os.Stderr, " Kubeconfig=%v\n", Kubeconfig) + fmt.Fprintf(os.Stderr, " Go=%v\n", Go) + fmt.Fprintf(os.Stderr, " Git=%v\n", Git) + fmt.Fprintf(os.Stderr, " Verbose=%v\n", Verbose) + + // Coverage + // -------- + // Create Gocoverdir if it does not already exist + // FIXME + + // Version + fmt.Fprintln(os.Stderr, "---------------------") + fmt.Fprintln(os.Stderr, "Func Version:") + printVersion() + + fmt.Fprintln(os.Stderr, "--- init complete ---") + fmt.Fprintln(os.Stderr, "") // TODO: there is a superfluous linebreak from "func version". This balances the whitespace. +} + +// readEnvs and apply defaults, populating the named global variables with +// the final values which will be used by all tests. +func readEnvs() { // Bin - path to binary which will be used when running the tests. - Bin = os.Getenv("E2E_FUNC_BIN") // Read in deprecated env first - if Bin != "" { // warn if found - fmt.Fprintln(os.Stderr, "WARNING: The env var E2E_FUNC_BIN is deprecated and support will be removed in a future release. Please use FUNC_E2E_BIN.") - } - if v := os.Getenv("FUNC_E2E_BIN"); v != "" { // overwrite with current env - Bin = v - } - if Bin == "" { // Default - Bin = DefaultBin - } - if !filepath.IsAbs(Bin) { // convert to abs - v, err := filepath.Abs(Bin) - if err != nil { - panic(fmt.Sprintf("error converting path to absolute. %v", err)) - } - Bin = v - } - fmt.Fprintf(os.Stderr, " Bin=%v\n", Bin) // echo for verification + // Args: current ENV, deprecated ENV, default. + Bin = getEnvPath("FUNC_E2E_BIN", "E2E_FUNC_BIN", DefaultBin) // Plugin - if set, func is a plugin and Bin is the one plugging. The value // is the name of the subcommand. If set to "true", for backwards compat // the default value is "func" - Plugin = os.Getenv("E2E_USE_KN_FUNC") // read in the deprecated env - if Plugin == "true" { - fmt.Fprintln(os.Stderr, "WARNING: The env var E2E_USE_KN_FUNC is deprecated and support will be removed in a future release. Please use FUNC_E2E_PLUGIN and set to the value 'func'.") - // "true" is for backwards compatibility. - // The new env var is a string indicating the name of the plugin's - // subcommand, which for that case was always "func" - Plugin = "func" - } - if v := os.Getenv("FUNC_E2E_PLUGIN"); v != "" { // override with new - Plugin = v + Plugin = getEnv("FUNC_E2E_PLUGIN", "E2E_USE_KN_FUNC", "") + if Plugin == "true" { // backwards compatibility + Plugin = "func" // deprecated value was literal string "true" } - fmt.Fprintf(os.Stderr, " Plugin=%v\n", Plugin) // echo // Registry - the registry URL including any account/repository at that // registry. Example: docker.io/alice. Default is the local registry. - Registry = os.Getenv("E2E_REGISTRY_URL") // read in the deprecated env - if Registry != "" { - fmt.Fprintln(os.Stderr, "WARNING: the env var E2E_REGISTRY_URL is deprecated and support will be removed in a future release. Please use FUNC_E2E_REGISTRY.") - } - if v := os.Getenv("FUNC_E2E_REGISTRY"); v != "" { // overwrite with new - Registry = v - } - if Registry == "" { // default - Registry = DefaultRegistry - } - fmt.Fprintf(os.Stderr, " Registry=%v\n", Registry) // echo + Registry = getEnv("FUNC_E2E_REGISTRY", "E2E_REGISTRY_URL", DefaultRegistry) // Runtimes - can optionally pass a list of runtimes to test, overriding // the default of testing all builtin runtimes. // Example "FUNC_E2E_RUNTIMES=go,python" - runtimes := os.Getenv("E2E_RUNTIMES") - if runtimes != "" { - fmt.Fprintln(os.Stderr, "WARNING: the env var E2E_RUNTIMES is deprecated and support will be removed in a future release. Please use FUNC_E2E_RUNTIMES and set to a comma-delimited list.") - Runtimes = fromCSV(runtimes) - } - if runtimes = os.Getenv("FUNC_E2E_RUNTIMES"); runtimes != "" { - Runtimes = fromCSV(runtimes) - } - if len(Runtimes) == 0 { - Runtimes = DefaultRuntimes - } - fmt.Fprintf(os.Stderr, " Runtimes=%v\n", toCSV(Runtimes)) + Runtimes = getEnvList("FUNC_E2E_RUNTIMES", "E2E_RUNTIMES", "") // Builders - can optionally pass a list of builders to test, overriding // the default of testing all. Example "FUNC_E2E_BUILDERS=pack,s2i" - if builders := os.Getenv("FUNC_E2E_BUILDERS"); builders != "" { - Builders = fromCSV(builders) - } - if len(Builders) == 0 { - Builders = DefaultBuilders - } - fmt.Fprintf(os.Stderr, " Builders=%v\n", toCSV(Builders)) + Builders = getEnvList("FUNC_E2E_BUILDERS", "", "") // Kubeconfig - the kubeconfig to pass ass KUBECONFIG env to test // environments. - Kubeconfig = os.Getenv("FUNC_E2E_KUBECONFIG") - if Kubeconfig == "" { - Kubeconfig = DefaultKubeconfig - } - if !filepath.IsAbs(Kubeconfig) { // convert to abs - v, err := filepath.Abs(Kubeconfig) - if err != nil { - panic(fmt.Sprintf("error converting path to absolute. %v", err)) - } - Kubeconfig = v - } - fmt.Fprintf(os.Stderr, " Kubeconfig=%v\n", Kubeconfig) // echo + Kubeconfig = getEnvPath("FUNC_E2E_KUBECONFIG", "", DefaultKubeconfig) // Gocoverdir - the coverage directory to use while testing the go binary. - Gocoverdir = os.Getenv("FUNC_E2E_GOCOVERDIR") - if Gocoverdir == "" { - Gocoverdir = DefaultGocoverdir - } - if !filepath.IsAbs(Gocoverdir) { // convert to abs - v, err := filepath.Abs(Gocoverdir) - if err != nil { - panic(fmt.Sprintf("error converting path to absolute. %v", err)) - } - Gocoverdir = v - } - fmt.Fprintf(os.Stderr, " Gocoverdir=%v\n", Gocoverdir) // echo + Gocoverdir = getEnvPath("FUNC_E2E_GOCOVERDIR", "", DefaultGocoverdir) + + // Go binary path + Go = getEnvBin("FUNC_E2E_GO", "", "go") + + // Git binary path + Git = getEnvBin("FUNC_E2E_GIT", "", "git") + + // Verbose env as a truthy boolean + Verbose = getEnvBool("FUNC_E2E_VERBOSE", "", DefaultVerbose) - // Home is the default home directory, is not configurable (tests override - // it on a case-by-case basis) and is merely set here to the absolute path - // to DefaultHome (./testdata/default_home) + // Home is a bit of a special case. It is the default home directory, is + // not configurable (tests override it on a case-by-case basis) and is + // merely set here to the absolute path of DefaultHome var err error if Home, err = filepath.Abs(DefaultHome); err != nil { panic(fmt.Sprintf("error converting the relative default home value to absolute. %v", err)) } +} - Go = os.Getenv("FUNC_E2E_GO") - if Go == "" { - goBin, err := exec.LookPath("go") - if err != nil { - panic(fmt.Sprintf("error locating to 'go' executable. %v", err)) - } - Go = goBin - } - if !filepath.IsAbs(Go) { // convert to abs - v, err := filepath.Abs(Go) - if err != nil { +// getEnvPath converts the value returned from getEnv to an absolute path. +// See getEnv docs for details. +func getEnvPath(env, deprecated, dflt string) (val string) { + val = getEnv(env, deprecated, dflt) + if !filepath.IsAbs(val) { // convert to abs + var err error + if val, err = filepath.Abs(val); err != nil { panic(fmt.Sprintf("error converting path to absolute. %v", err)) } - Go = v } - fmt.Fprintf(os.Stderr, " Go=%v\n", Go) // echo + return +} - // Go binary path +// getEnvPath converts the value returned from getEnv into a string slice. +func getEnvList(env, deprecated, dflt string) (vals []string) { + return fromCSV(getEnv(env, deprecated, dflt)) +} - // Coverage - // -------- - // Create Gocoverdir if it does not already exist - // FIXME +// getEnvBool converts the value returned from getEnv into a boolean. +func getEnvBool(env, deprecated string, dfltBool bool) bool { + dflt := fmt.Sprintf("%t", dfltBool) + val, err := strconv.ParseBool(getEnv(env, deprecated, dflt)) + if err != nil { + panic(fmt.Sprintf("value for %v %v expected to be boolean. %v", env, deprecated, err)) + } + return val +} - // Version - // ------- - // Print version of func which is being used, taking into account if - // we're running as a plugin. - fmt.Fprintln(os.Stderr, "---------------------") - fmt.Fprintln(os.Stderr, "Testing Func Version:") +// getEnvBin converts the value returned from getEnv into an absolute path. +// and if not provided checks the current PATH for a matching binary name, +// and returns the absolute path to that. +func getEnvBin(env, deprecated, dflt string) string { + val, err := exec.LookPath(getEnv(env, deprecated, dflt)) + if err != nil { + fmt.Fprintf(os.Stderr, "error locating command %q. %v", val, err) + } + return val +} + +// getEnv gets the value of the given environment variable, or the default. +// If the optional deprecated environment variable name is passed, it will be used +// as a fallback with a warning about its deprecation status being printed. +// The final value will be converted to an absolute path. +func getEnv(env, deprecated, dflt string) (val string) { + // First check deprecated if provided + if deprecated != "" { + if val = os.Getenv(deprecated); val != "" { + fmt.Fprintf(os.Stderr, "warning: the env var %v is deprecated and support will be removed in a future release. please use %v.", deprecated, env) + } + } + // Current env takes precidence + if v := os.Getenv(env); v != "" { + val = v + } + // Default + if val == "" { + val = dflt + } + return +} + +// printVersion of func which is being used, taking into account if +// we're running as a plugin. +func printVersion() { args := []string{"version", "--verbose"} bin := Bin if Plugin != "" { @@ -513,67 +614,56 @@ func init() { if err := cmd.Run(); err != nil { os.Exit(1) } - - fmt.Fprintln(os.Stderr, "--- init complete ---") - fmt.Fprintln(os.Stderr, "") // TODO: there is a superfluous linebreak on "func version". This balances the whitespace. } -// resetEnv removes environment variables from the process. +// ---------------------------------------------------------------------------- +// Test Helpers +// ---------------------------------------------------------------------------- + +// resetEnv before running a test to remove all environment variables and +// set the required environment variables to those specified during +// initialization. // // Every test must be run with a nearly completely isolated environment, -// otherwise a developer's local environment will affect the E2E tests when run -// locally outside of CI. -// -// Some environment variables, provided via FUNC_E2E_* or other settings, -// are explicitly set here. -// -// For example, the system requires HOME to be set (WIP to remove requirement) -// so HOME is explicitly set to ./testdata/default_home, to be overridden -// as needed by tests which require specific home configuraitons for their -// execution. +// otherwise a developer's local environment will affect the E2E tests when +// run locally outside of CI. Some environment variables, provided via +// FUNC_E2E_* or other settings, are explicitly set here. func resetEnv() { - // // Clear all except for those whitelisted - // options := []string{ - // "FUNC_E2E_EXAMPLE_SETTING", // TODO: remove if not used - // } - // for _, env := range os.Environ() { - // pair := strings.SplitN(env, "=", 2) - // // t.Logf("unsetenv %v\n", pair) - // if slices.Contains(options, pair[0]) { - // continue - // } - // os.Unsetenv(pair[0]) - // t.Cleanup(func() { - // if len(pair) == 2 { // t.Logf("setenv %v=%v\n", pair[0], pair[1]) - // os.Setenv(pair[0], pair[1]) - // } else if len(pair) == 1 { - // // t.Logf("setenv %v\n", pair[0]) - // os.Setenv(pair[0], "") - // } else { - // panic(fmt.Sprintf("unexpected env length %v for env %v.", len(pair), env)) - // } - // }) - // } os.Clearenv() - os.Setenv("KUBECONFIG", Kubeconfig) - os.Setenv("GOCOVERDIR", Gocoverdir) os.Setenv("HOME", Home) - // Host builder is currently behind a feature flag. The core tests rely - // on it completely. - os.Setenv("FUNC_ENABLE_HOST_BUILDER", "true") + os.Setenv("KUBECONFIG", Kubeconfig) os.Setenv("FUNC_GO", Go) + os.Setenv("FUNC_GIT", Git) + os.Setenv("GOCOVERDIR", Gocoverdir) + os.Setenv("FUNC_VERBOSE", fmt.Sprintf("%t", Verbose)) + + // The Registry will be set either during first-time setup using the + // global config, or already defaulted by the user via environment variable. + os.Setenv("FUNC_REGISTRY", Registry) + + // The following host-builder related settings will become the defaults + // once the host builder supports the core runtimes. Setting them here in + // order to futureproof individual tests. + os.Setenv("FUNC_ENABLE_HOST_BUILDER", "true") // Enable the host builder + os.Setenv("FUNC_BUILDER", "host") // default to host builder + os.Setenv("FUNC_CONTAINER", "false") // "run" uses host builder } // cdTmp changes to a new temporary directory for running the test. -// Essentially equvalent to creating a new directory before beginning to +// Essentially equivalent to creating a new directory before beginning to // use func. The path created is returned. -func cdTemp(t *testing.T) string { +// The "name" argument is the name of the final Function's directory. +// Note that this will be unnecessary when upcoming changes remove the logic +// which uses the current directory name by default for funciton name and +// instead requires an explicit name argument on build/deploy. +// Name should be unique per test to enable better test isolation. +func cdTemp(t *testing.T, name string) string { pwd, err := os.Getwd() if err != nil { panic(err) } - tmp := filepath.Join(t.TempDir(), DefaultName) - if err := os.MkdirAll(tmp, 0744); err != nil { + tmp := filepath.Join(t.TempDir(), name) + if err := os.MkdirAll(tmp, 0755); err != nil { panic(err) } if err := os.Chdir(tmp); err != nil { @@ -624,6 +714,112 @@ func newCmd(t *testing.T, args ...string) *exec.Cmd { // return stdout.String() } +// waitFor returns true if there is service at the given addresss which +// echoes back the request arguments given. +// +// TODO: Implement a --output=json flag on `func run` and update all +// callers currently passing localhost:8080 with this calculated value. +// +// Reasoning: This will be a false negative if port 8080 is being used +// by another proces. This will fail because, if a service is already running +// on port 8080, Functions will automatically choose to run the next higher +// unused port. And this will be a false positive if there happens to be +// a service not already running on the port which happens to implement an +// echo. For example there is another process outside the E2Es which is +// currently executing a `func run` +// Note that until this is implemented, this temporary implementation also +// forces single-threaded test execution. +func waitFor(t *testing.T, address string) (ok bool) { + t.Helper() + retries := 50 // Set fairly high for slow environments such as free-tier CI + warnThreshold := 30 // start warning after 30 + warnModulo := 5 // but only warn every 5 attemtps + delay := 500 * time.Millisecond + for i := 0; i < retries; i++ { + time.Sleep(delay) + res, err := http.Get(address + "?test-echo-param") + if err != nil { + if i > warnThreshold && i%warnModulo == 0 { + t.Logf("unable to contact function (attempt %v/%v). %v", i, retries, err) + } + continue + } + body, err := io.ReadAll(res.Body) + if err != nil { + t.Logf("error reading function response. %v", err) + continue + } + defer res.Body.Close() + if strings.Index(string(body), "test-echo-param") == -1 { + t.Log("Response received, but it does not appear to be an echo.") + t.Log("Full dump:") + dump, _ := httputil.DumpResponse(res, true) + t.Log(string(dump)) + continue + } + return true + } + t.Logf("Could not contact function after %v tries", retries) + return +} + +// waitForContent returns true if there is a service listening at the +// given addresss which responds HTTP OK with the given string in its body. +func waitForContent(t *testing.T, address, content string) (ok bool) { + t.Helper() + retries := 50 // Set fairly high for slow environments such as free-tier CI + warnThreshold := 30 // start warning after 30 + warnModulo := 5 // but only warn every 5 attemtps + delay := 500 * time.Millisecond + for i := 0; i < retries; i++ { + time.Sleep(delay) + res, err := http.Get(address) + if err != nil { + if i > warnThreshold && i%warnModulo == 0 { + t.Logf("unable to contact function (attempt %v/%v). %v", i, retries, err) + } + continue + } + body, err := io.ReadAll(res.Body) + if err != nil { + t.Logf("error reading function response. %v", err) + continue + } + defer res.Body.Close() + if !strings.Contains(string(body), content) { + t.Log("Response received, but it did not contain the expected content.") + t.Log("Full dump:") + dump, _ := httputil.DumpResponse(res, true) + t.Log(string(dump)) + continue + } + return true + } + t.Logf("Could not validate function returns expected content after %v tries", retries) + return + +} + +// isAbnormalExit checks an erro returned from a cmd.Wait and returns true +// Removed +// if the error is something other than a known exit 130 from a SIGINT. +func isAbnormalExit(t *testing.T, err error) bool { + t.Helper() + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode := exitErr.ExitCode() + // When interrupted, the exit will exit with an ExitError, but + // should be exit code 130 (the code for SIGINT) + if exitCode != 0 && exitCode != 130 { + t.Fatalf("Function exited code %v", exitErr.ExitCode()) + return true + } + } else { + t.Fatalf("Function errored during execution. %v", err) + return true + } + return false +} + func fromCSV(s string) (result []string) { result = []string{} ss := strings.Split(s, ",")