Merge pull request #2255 from smg247/native-skips

TRT-2066: Use OTE generated labels for skips in test-kubernetes-e2e.sh

Author: openshift-merge-bot[bot]

.openshift-tests-extension/openshift_payload_hyperkube.json

@@ -80076,4 +80076,4 @@
     "source": "openshift:payload:hyperkube",
     "lifecycle": "blocking"
   }
-]
+]

openshift-hack/cmd/k8s-tests-ext/environment_selectors.go

@@ -1,6 +1,10 @@
 package main
 
-import et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+import (
+	"fmt"
+
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+)
 
 // addEnvironmentSelectors adds the environmentSelector field to appropriate specs to facilitate including or excluding
 // them based on attributes of the cluster they are running on
@@ -16,11 +20,11 @@ func addEnvironmentSelectors(specs et.ExtensionTestSpecs) {
 	specs.SelectAny([]et.SelectFunction{ // Since these must use "NameContainsAll" they cannot be included in filterByPlatform
 		et.NameContainsAll("[sig-network] LoadBalancers [Feature:LoadBalancer]", "UDP"),
 		et.NameContainsAll("[sig-network] LoadBalancers [Feature:LoadBalancer]", "session affinity"),
-	}).Exclude(et.PlatformEquals("aws"))
+	}).Exclude(et.PlatformEquals("aws")).AddLabel("[Skipped:aws]")
 
 	specs.SelectAny([]et.SelectFunction{ // Since these must use "NameContainsAll" they cannot be included in filterByNetwork
 		et.NameContainsAll("NetworkPolicy", "named port"),
-	}).Exclude(et.NetworkEquals("OVNKubernetes"))
+	}).Exclude(et.NetworkEquals("OVNKubernetes")).AddLabel("[Skipped:Network/OVNKubernetes]")
 }
 
 // filterByPlatform is a helper function to do, simple, "NameContains" filtering on tests by platform
@@ -127,7 +131,9 @@ func filterByPlatform(specs et.ExtensionTestSpecs) {
 			selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 		}
 
-		specs.SelectAny(selectFunctions).Exclude(et.PlatformEquals(platform))
+		specs.SelectAny(selectFunctions).
+			Exclude(et.PlatformEquals(platform)).
+			AddLabel(fmt.Sprintf("[Skipped:%s]", platform))
 	}
 }
 
@@ -169,7 +175,9 @@ func filterByExternalConnectivity(specs et.ExtensionTestSpecs) {
 			selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 		}
 
-		specs.SelectAny(selectFunctions).Exclude(et.ExternalConnectivityEquals(externalConnectivity))
+		specs.SelectAny(selectFunctions).
+			Exclude(et.ExternalConnectivityEquals(externalConnectivity)).
+			AddLabel(fmt.Sprintf("[Skipped:%s]", externalConnectivity))
 	}
 }
 
@@ -198,7 +206,9 @@ func filterByTopology(specs et.ExtensionTestSpecs) {
 			selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 		}
 
-		specs.SelectAny(selectFunctions).Exclude(et.TopologyEquals(topology))
+		specs.SelectAny(selectFunctions).
+			Exclude(et.TopologyEquals(topology)).
+			AddLabel(fmt.Sprintf("[Skipped:%s]", topology))
 	}
 }
 
@@ -217,7 +227,9 @@ func filterByNoOptionalCapabilities(specs et.ExtensionTestSpecs) {
 	for _, exclusion := range exclusions {
 		selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 	}
-	specs.SelectAny(selectFunctions).Exclude(et.NoOptionalCapabilitiesExist())
+	specs.SelectAny(selectFunctions).
+		Exclude(et.NoOptionalCapabilitiesExist()).
+		AddLabel("[Skipped:NoOptionalCapabilities]")
 }
 
 // filterByNetwork is a helper function to do, simple, "NameContains" filtering on tests by network
@@ -230,6 +242,8 @@ func filterByNetwork(specs et.ExtensionTestSpecs) {
 			selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 		}
 
-		specs.SelectAny(selectFunctions).Exclude(et.NetworkEquals(network))
+		specs.SelectAny(selectFunctions).
+			Exclude(et.NetworkEquals(network)).
+			AddLabel(fmt.Sprintf("[Skipped:%s]", network))
 	}
 }

openshift-hack/cmd/k8s-tests-ext/k8s-tests.go

@@ -4,7 +4,9 @@ import (
 	"flag"
 	"os"
 	"reflect"
+	"strconv"
 
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	"k8s.io/kubernetes/openshift-hack/e2e/annotate/generated"
 	"k8s.io/kubernetes/test/e2e/framework"
 
@@ -14,7 +16,6 @@ import (
 	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
 	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
-	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
 	v "github.com/openshift-eng/openshift-tests-extension/pkg/version"
 
@@ -97,11 +98,21 @@ func main() {
 	//		  the environmental skip code from the enhancement once its implemented.
 	//		- Make sure to account for test renames that occur because of removal of these
 	//		  annotations
-	specs.Walk(func(spec *et.ExtensionTestSpec) {
-		if annotations, ok := generated.Annotations[spec.Name]; ok {
-			spec.Name += annotations
+	var omitAnnotations bool
+	omitAnnotationsVal := os.Getenv("OMIT_ANNOTATIONS")
+	if omitAnnotationsVal != "" {
+		omitAnnotations, err = strconv.ParseBool(omitAnnotationsVal)
+		if err != nil {
+			panic("Failed to parse OMIT_ANNOTATIONS: " + err.Error())
 		}
-	})
+	}
+	if !omitAnnotations {
+		specs.Walk(func(spec *et.ExtensionTestSpec) {
+			if annotations, ok := generated.Annotations[spec.Name]; ok {
+				spec.Name += annotations
+			}
+		})
+	}
 
 	specs = filterOutDisabledSpecs(specs)
 	addLabelsToSpecs(specs)

openshift-hack/cmd/k8s-tests-ext/labels.go

@@ -7,7 +7,7 @@ import (
 func addLabelsToSpecs(specs et.ExtensionTestSpecs) {
 	var namesByLabel = map[string][]string{
 		// tests too slow to be part of conformance
-		"Slow": {
+		"[Slow]": {
 			"[sig-scalability]",                            // disable from the default set for now
 			"should create and stop a working application", // Inordinately slow tests
 
@@ -16,13 +16,13 @@ func addLabelsToSpecs(specs et.ExtensionTestSpecs) {
 			"validates that there exists conflict between pods with same hostPort and protocol but one using 0.0.0.0 hostIP", // 5m, really?
 		},
 		// tests that are known flaky
-		"Flaky": {
+		"[Flaky]": {
 			"Job should run a job to completion when tasks sometimes fail and are not locally restarted", // seems flaky, also may require too many resources
 			// TODO(node): test works when run alone, but not in the suite in CI
 			"[Feature:HPA] Horizontal pod autoscaling (scale resource: CPU) [sig-autoscaling] ReplicationController light Should scale from 1 pod to 2 pods",
 		},
 		// tests that must be run without competition
-		"Serial": {
+		"[Serial]": {
 			"[Disruptive]",
 			"[Feature:Performance]", // requires isolation
 

openshift-hack/e2e/kube_e2e_test.go

@@ -1,22 +1,22 @@
 package e2e
 
-//go:generate go run -mod vendor ./annotate/cmd -- ./annotate/generated/zz_generated.annotations.go
-
 // This file duplicates most of test/e2e/e2e_test.go but limits the included
 // tests (via include.go) to tests that are relevant to openshift.
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"math/rand"
 	"os"
+	"os/exec"
 	"strings"
 	"testing"
 	"time"
 
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	"gopkg.in/yaml.v2"
-
 	// Never, ever remove the line with "/ginkgo". Without it,
 	// the ginkgo test runner will not detect that this
 	// directory contains a Ginkgo test suite.
@@ -35,9 +35,6 @@ import (
 	e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests"
 	testfixtures "k8s.io/kubernetes/test/fixtures"
 	"k8s.io/kubernetes/test/utils/image"
-
-	// Ensure test annotation
-	"k8s.io/kubernetes/openshift-hack/e2e/annotate/generated"
 )
 
 func TestMain(m *testing.M) {
@@ -109,17 +106,43 @@ func TestMain(m *testing.M) {
 }
 
 func TestE2E(t *testing.T) {
-	// TODO(soltysh): this is raw copy from end of openshift-hack/e2e/annotate/generated/zz_generated.annotations.go
-	// https://issues.redhat.com/browse/OCPBUGS-25641
+	// In order to properly skip tests, we must add the labels that the OTE external binary supplies to the test name
+	// This will then be used by Ginkgo to skip specific tests
+	oteCmd := exec.Command("k8s-tests-ext", "list", "tests")
+	// We can't have OTE also add annotations to the spec names to map to labels, or they won't match the actual spec names
+	//TODO(sgoeddel): once annotation logic is removed, this can be as well
+	oteCmd.Env = append(oteCmd.Env, "OMIT_ANNOTATIONS=true")
+	oteCmd.Stderr = os.Stderr
+	output, err := oteCmd.Output()
+	if err != nil {
+		t.Fatalf("Error running ote list tests command: %v", err)
+	}
+	var specs et.ExtensionTestSpecs
+	if err = json.Unmarshal(output, &specs); err != nil {
+		t.Fatalf("Error parsing ote list tests output: %v", err)
+	}
+
+	nameToLabels := make(map[string][]string, len(specs))
+	for _, spec := range specs {
+		nameToLabels[spec.Name] = spec.Labels.UnsortedList()
+	}
+
 	ginkgo.GetSuite().SetAnnotateFn(func(name string, node types.TestSpec) {
-		if newLabels, ok := generated.Annotations[name]; ok {
-			node.AppendText(newLabels)
+		if newLabels, ok := nameToLabels[name]; ok {
+			for _, label := range newLabels {
+				// Only add the label to the name if it isn't already present to avoid test names that are too long
+				if !strings.Contains(name, label) {
+					node.AppendText(fmt.Sprintf(" %s", label))
+				}
+			}
 		} else {
-			panic(fmt.Sprintf("unable to find test %s", name))
+			// If the name isn't found in the mapping, it is because the test has been disabled via OTE
+			node.AppendText(" [Disabled:missing]")
 		}
 		if strings.Contains(name, "Kubectl client Kubectl prune with applyset should apply and prune objects") {
 			fmt.Printf("Trying to annotate %q\n", name)
 		}
+
 	})
 
 	e2e.RunE2ETests(t)

openshift-hack/images/tests/Dockerfile.rhel

@@ -3,15 +3,18 @@ WORKDIR /go/src/k8s.io/kubernetes
 COPY . .
 RUN make WHAT=openshift-hack/e2e/k8s-e2e.test; \
     make WHAT=vendor/github.com/onsi/ginkgo/v2/ginkgo; \
+    make WHAT=openshift-hack/cmd/k8s-tests-ext; \
     mkdir -p /tmp/build; \
     cp /go/src/k8s.io/kubernetes/_output/local/bin/linux/$(go env GOARCH)/k8s-e2e.test /tmp/build/; \
     cp /go/src/k8s.io/kubernetes/_output/local/bin/linux/$(go env GOARCH)/ginkgo /tmp/build/; \
+    cp /go/src/k8s.io/kubernetes/_output/local/bin/linux/$(go env GOARCH)/k8s-tests-ext /tmp/build/; \
     cp /go/src/k8s.io/kubernetes/openshift-hack/test-kubernetes-e2e.sh /tmp/build/; \
     cp /go/src/k8s.io/kubernetes/openshift-hack/images/kube-proxy/test-kube-proxy.sh /tmp/build/
 
 FROM registry.ci.openshift.org/ocp/4.19:tools
 COPY --from=builder /tmp/build/k8s-e2e.test /usr/bin/
 COPY --from=builder /tmp/build/ginkgo /usr/bin/
+COPY --from=builder /tmp/build/k8s-tests-ext /usr/bin/
 COPY --from=builder /tmp/build/test-kubernetes-e2e.sh /usr/bin/
 COPY --from=builder /tmp/build/test-kube-proxy.sh /usr/bin/
 RUN yum install --setopt=tsflags=nodocs -y git gzip util-linux && yum clean all && rm -rf /var/cache/yum/* && \

openshift-hack/test-kubernetes-e2e.sh

@@ -54,12 +54,13 @@ esac
 # -skip and -focus.
 KUBE_E2E_TEST_ARGS="${KUBE_E2E_TEST_ARGS:-${DEFAULT_TEST_ARGS}}"
 
-# k8s-e2e.test and ginkgo are expected to be in the path in
+# k8s-e2e.test, ginkgo, and k8s-tests-ext are expected to be in the path in
 # CI. Outside of CI, ensure k8s-e2e.test and ginkgo are built and
 # available in PATH.
 if ! which k8s-e2e.test &> /dev/null; then
   make WHAT=vendor/github.com/onsi/ginkgo/v2/ginkgo
   make WHAT=openshift-hack/e2e/k8s-e2e.test
+  make WHAT=openshift-hack/cmd/k8s-tests-ext
   ROOT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd -P)"
   PATH="${ROOT_PATH}/_output/local/bin/$(go env GOHOSTOS)/$(go env GOARCH):${PATH}"
   export PATH