Separate flaky and non-flaky

Signed-off-by: apostasie <[email protected]>

Dockerfile

@@ -276,7 +276,8 @@ ARG DEBIAN_FRONTEND=noninteractive
 # `expect` package contains `unbuffer(1)`, which is used for emulating TTY for testing
 RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
     expect \
-    git
+    git \
+    make
 COPY --from=goversion /GOVERSION /GOVERSION
 ARG TARGETARCH
 RUN curl -fsSL --proto '=https' --tlsv1.2 https://golang.org/dl/$(cat /GOVERSION).linux-${TARGETARCH:-amd64}.tar.gz | tar xzvC /usr/local
@@ -314,8 +315,6 @@ RUN curl -o nydus-static.tgz -fsSL --proto '=https' --tlsv1.2 "https://github.co
   tar xzf nydus-static.tgz && \
   mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \
   rm nydus-static.tgz
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
 FROM test-integration AS test-integration-rootless
 # Install SSH for creating systemd user session.
@@ -338,17 +337,10 @@ RUN systemctl disable test-integration-ipfs-offline
 VOLUME /home/rootless/.local/share
 COPY ./Dockerfile.d/test-integration-rootless.sh /
 RUN chmod a+rx /test-integration-rootless.sh
-CMD ["/test-integration-rootless.sh", \
-  "gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
 # test for CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns
 FROM test-integration-rootless AS test-integration-rootless-port-slirp4netns
 COPY ./Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf /home/rootless/.config/systemd/user/containerd.service.d/port-slirp4netns.conf
 RUN chown -R rootless:rootless /home/rootless/.config
 
-FROM test-integration AS test-integration-ipv6
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-ipv6"]
-
 FROM base AS demo

Makefile

@@ -86,6 +86,9 @@ lint-yaml:
 lint-shell: $(call recursive_wildcard,$(MAKEFILE_DIR)/,*.sh)
 	shellcheck -a -x $^
 
+test-unit:
+	go test -v $(MAKEFILE_DIR)/pkg/...
+
 binaries: nerdctl
 
 install:

hack/test-integration.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# shellcheck disable=SC2034,SC2015
+set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
+readonly root
+readonly timeout="60m"
+
+# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
+args=(--format=testname --jsonfile /tmp/test-integration.log --packages="$root"/../cmd/nerdctl/...)
+
+for arg in "$@"; do
+  if [ "$arg" == "-test.only-flaky" ]; then
+    args+=("--rerun-fails=2")
+    break
+  fi
+done
+
+gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@"
+
+echo "These are the tests that took more than 20 seconds:"
+gotestsum tool slowest --threshold 20s --jsonfile /tmp/test-integration.log

pkg/testutil/nerdtest/requirements.go

@@ -73,6 +73,11 @@ var OnlyKubernetes = &test.Requirement{
 		} else {
 			mess = "runner skips Kubernetes compatible tests in the non-Kubernetes environment"
 		}
+		_, err := exec.LookPath("kubectl")
+		if err != nil {
+			ret = false
+			mess = fmt.Sprintf("kubectl is not in the path: %+v", err)
+		}
 		return ret, mess
 	},
 }

pkg/testutil/testutil.go

@@ -787,6 +787,9 @@ func newBase(t *testing.T, ns string, ipv6Compatible bool, kubernetesCompatible
 	} else if !base.EnableKubernetes && base.KubernetesCompatible {
 		t.Skip("runner skips Kubernetes compatible tests in the non-Kubernetes environment")
 	}
+	if !GetFlakyEnvironment() && !GetEnableKubernetes() && !GetEnableIPv6() {
+		t.Skip("legacy tests are considered flaky by default and are skipped unless in the flaky environment")
+	}
 	var err error
 	switch base.Target {
 	case Nerdctl: