diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 0d7eb98572..e571e4f118 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -118,19 +118,19 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CI=true working-directory: ./tests - name: Run functional graceful-recovery tests run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests - name: Run functional tests run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CI=true working-directory: ./tests diff --git a/Makefile b/Makefile index 231f2af933..2f1cf7fdb0 100644 --- a/Makefile +++ b/Makefile @@ -45,8 +45,8 @@ endif .PHONY: help help: Makefile ## Display this help - @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' - @grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' + @grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' .PHONY: build-prod-images build-prod-images: build-prod-ngf-image build-prod-nginx-image ## Build the NGF and nginx docker images for production diff --git a/tests/Makefile b/tests/Makefile index e9cc59004a..87e5379fa7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,10 +1,11 @@ -CI=false +CI ?= false CLUSTER_NAME ?= kind CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image CONFORMANCE_TAG = latest## Tag for the conformance test runner image GATEWAY_CLASS = nginx## Gateway class to use GINKGO_FLAGS = GINKGO_LABEL = +GITHUB_OUTPUT = GW_API_VERSION ?= $(shell sed -n 's/.*ref=v\(.*\)/\1/p' ../config/crd/gateway-api/standard/kustomization.yaml)## Supported Gateway API version from current NGF GW_API_PREV_VERSION ?= 1.1.0## Supported Gateway API version from previous NGF release GW_SERVICE_TYPE = NodePort## Service type to use for the gateway @@ -15,7 +16,11 @@ PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml SUPPORTED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,GRPCExactMethodMatching,GRPCRouteListenerHostnameMatching,GRPCRouteHeaderMatching ifneq ($(GINKGO_LABEL),) - override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)" + override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)" +endif + +ifeq ($(CI),true) + GITHUB_OUTPUT := --github-output endif .PHONY: update-go-modules @@ -33,9 +38,9 @@ run-conformance-tests: ## Run conformance tests kubectl apply -f conformance/conformance-rbac.yaml kubectl run -i conformance \ --image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \ - --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ + --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ --restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \ - --supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \ + --supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \ --report-output=output.txt; cat output.txt" | tee output.txt ./scripts/check-pod-exit-code.sh sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml @@ -65,8 +70,8 @@ reset-go-modules: ## Reset the go modules changes # Check if PLUS_ENABLED is true ifeq ($(PLUS_ENABLED),true) - # If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX - NGINX_PREFIX := $(NGINX_PLUS_PREFIX) + # If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX + NGINX_PREFIX := $(NGINX_PLUS_PREFIX) endif .PHONY: setup-gcp-and-run-tests @@ -107,10 +112,11 @@ start-longevity-test: nfr-test ## Start the longevity test to run for 4 days in stop-longevity-test: export STOP_LONGEVITY=true stop-longevity-test: nfr-test ## Stop the longevity test and collects results + .PHONY: .vm-nfr-test .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`) - go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines $(ifeq $(CI),true,--github-output) \ - --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite -- --gateway-api-version=$(GW_API_VERSION) \ + go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \ + --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite -- --gateway-api-version=$(GW_API_VERSION) \ --gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ --plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ --pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \ @@ -118,7 +124,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results .PHONY: test test: ## Runs the functional tests on your default k8s cluster - go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines \ + go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \ --label-filter "functional" $(GINKGO_FLAGS) ./suite -- \ --gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \ --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ diff --git a/tests/framework/info.go b/tests/framework/info.go index e83a1be182..588b728631 100644 --- a/tests/framework/info.go +++ b/tests/framework/info.go @@ -2,11 +2,13 @@ package framework import ( "fmt" + "runtime/debug" core "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) +// GetLogs returns the logs for all containers in all pods for a release. func GetLogs(rm ResourceManager, namespace string, releaseName string) string { var returnLogs string pods, err := rm.GetPods(namespace, client.MatchingLabels{ @@ -32,6 +34,7 @@ func GetLogs(rm ResourceManager, namespace string, releaseName string) string { return returnLogs } +// GetEvents returns the events for a namespace. func GetEvents(rm ResourceManager, namespace string) string { var returnEvents string events, err := rm.GetEvents(namespace) @@ -53,3 +56,28 @@ func GetEvents(rm ResourceManager, namespace string) string { } return returnEvents } + +// GetBuildInfo returns the build information. +func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) { + commitHash = "unknown" + commitTime = "unknown" + dirtyBuild = "unknown" + + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + + for _, kv := range info.Settings { + switch kv.Key { + case "vcs.revision": + commitHash = kv.Value + case "vcs.time": + commitTime = kv.Value + case "vcs.modified": + dirtyBuild = kv.Value + } + } + + return +} diff --git a/tests/framework/results.go b/tests/framework/results.go index ac65569d11..79bc594c10 100644 --- a/tests/framework/results.go +++ b/tests/framework/results.go @@ -49,10 +49,13 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error { if ci.IsGKE { clusterType = "GKE" } + + commit, date, dirty := GetBuildInfo() + + //nolint:lll text := fmt.Sprintf( - //nolint:lll - "# Results\n\n## Test environment\n\nNGINX Plus: %v\n\n%s Cluster:\n\n- Node count: %d\n- k8s version: %s\n- vCPUs per node: %d\n- RAM per node: %s\n- Max pods per node: %d\n", - plus, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode, + "# Results\n\n## Test environment\n\nNGINX Plus: %v\n\nNGINX Gateway Fabric:\n\n- Commit: %s\n- Date: %s\n- Dirty: %v\n\n%s Cluster:\n\n- Node count: %d\n- k8s version: %s\n- vCPUs per node: %d\n- RAM per node: %s\n- Max pods per node: %d\n", + plus, commit, date, dirty, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode, ) if _, err := fmt.Fprint(file, text); err != nil { return err diff --git a/tests/scripts/remote-scripts/run-nfr-tests.sh b/tests/scripts/remote-scripts/run-nfr-tests.sh index d2db339478..89eac20e1b 100755 --- a/tests/scripts/remote-scripts/run-nfr-tests.sh +++ b/tests/scripts/remote-scripts/run-nfr-tests.sh @@ -10,7 +10,7 @@ elif [ "${STOP_LONGEVITY}" == "true" ]; then GINKGO_LABEL="longevity-teardown" fi -cd nginx-gateway-fabric/tests && make .vm-nfr-test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" +cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" if [ "${START_LONGEVITY}" == "true" ]; then suite/scripts/longevity-wrk.sh diff --git a/tests/scripts/remote-scripts/run-tests.sh b/tests/scripts/remote-scripts/run-tests.sh index db681d2e32..a15ea5cbc2 100755 --- a/tests/scripts/remote-scripts/run-tests.sh +++ b/tests/scripts/remote-scripts/run-tests.sh @@ -4,4 +4,4 @@ set -e source "${HOME}"/vars.env -cd nginx-gateway-fabric/tests && make test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" +cd nginx-gateway-fabric/tests && make test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh index 91c30b0ec1..77dc6f4761 100755 --- a/tests/scripts/run-tests-gcp-vm.sh +++ b/tests/scripts/run-tests-gcp-vm.sh @@ -16,6 +16,7 @@ gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SC gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ --command="export START_LONGEVITY=${START_LONGEVITY} &&\ export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ + export CI=${CI} &&\ bash -s" <"${SCRIPT_DIR}"/remote-scripts/${SCRIPT} retcode=$? diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index 26ef6fc4f8..baffadb0ef 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "bytes" diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index a1c3a6fb4f..703dbc3a21 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "fmt" diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index a9edbbe2d8..b7360175c1 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "context" diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index d42ebc8b26..4054ddb99c 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "fmt" diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go index d573c5903c..62d482edb2 100644 --- a/tests/suite/sample_test.go +++ b/tests/suite/sample_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "fmt" diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index e5737ad210..053d575d26 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "bytes" @@ -130,39 +130,39 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { } const scaleResultTemplate = ` - ## Test {{ .Name }} +## Test {{ .Name }} - ### Reloads +### Reloads - - Total: {{ .ReloadCount }} - - Total Errors: {{ .ReloadErrsCount }} - - Average Time: {{ .ReloadAvgTime }}ms - - Reload distribution: - {{- range .ReloadBuckets }} - - {{ .Le }}ms: {{ .Val }} - {{- end }} +- Total: {{ .ReloadCount }} +- Total Errors: {{ .ReloadErrsCount }} +- Average Time: {{ .ReloadAvgTime }}ms +- Reload distribution: +{{- range .ReloadBuckets }} + - {{ .Le }}ms: {{ .Val }} +{{- end }} - ### Event Batch Processing +### Event Batch Processing - - Total: {{ .EventsCount }} - - Average Time: {{ .EventsAvgTime }}ms - - Event Batch Processing distribution: - {{- range .EventsBuckets }} - - {{ .Le }}ms: {{ .Val }} - {{- end }} +- Total: {{ .EventsCount }} +- Average Time: {{ .EventsAvgTime }}ms +- Event Batch Processing distribution: +{{- range .EventsBuckets }} + - {{ .Le }}ms: {{ .Val }} +{{- end }} - ### Errors +### Errors - - NGF errors: {{ .NGFErrors }} - - NGF container restarts: {{ .NGFContainerRestarts }} - - NGINX errors: {{ .NginxErrors }} - - NGINX container restarts: {{ .NginxContainerRestarts }} +- NGF errors: {{ .NGFErrors }} +- NGF container restarts: {{ .NGFContainerRestarts }} +- NGINX errors: {{ .NginxErrors }} +- NGINX container restarts: {{ .NginxContainerRestarts }} - ### Graphs and Logs +### Graphs and Logs - See [output directory](./{{ .Name }}) for more details. - The logs are attached only if there are errors. - ` +See [output directory](./{{ .Name }}) for more details. +The logs are attached only if there are errors. +` writeScaleResults := func(dest io.Writer, results scaleTestResults) error { tmpl, err := template.New("results").Parse(scaleResultTemplate) @@ -905,7 +905,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim Expect(framework.WriteMetricsResults(outFile, res.metrics)).To(Succeed()) - link := fmt.Sprintf("\n\n![%[1]v.png](%[1]v.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme)) + link := fmt.Sprintf("\n\n![%[1]v-oss.png](%[1]v-oss.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme)) if *plusEnabled { link = fmt.Sprintf("\n\n![%[1]v-plus.png](%[1]v-plus.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme)) } diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 2ab0c4d45f..f0f3aa6993 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -1,4 +1,6 @@ -package suite +// This package needs to be named main to get build info +// because of https://github.com/golang/go/issues/33976 +package main import ( "context" diff --git a/tests/suite/telemetry_test.go b/tests/suite/telemetry_test.go index c6a8b65fc0..3ce1ec6669 100644 --- a/tests/suite/telemetry_test.go +++ b/tests/suite/telemetry_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "fmt" diff --git a/tests/suite/tracing_test.go b/tests/suite/tracing_test.go index 9876792c18..4e4519e3c5 100644 --- a/tests/suite/tracing_test.go +++ b/tests/suite/tracing_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "context" diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index 66c87922e8..1de1f45118 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "bytes"