Add podblocker test

Extends test coverage to ensure nothing breaks

Signed-off-by: Jean-Philippe Evrard <[email protected]>

.github/workflows/on-pr.yaml

@@ -127,6 +127,7 @@ jobs:
         testname:
           - "TestCordonningIsKept/concurrency1"
           - "TestCordonningIsKept/concurrency2"
+          - "TestE2EBlocker/podblocker"
     steps:
       - uses: actions/checkout@v4
       - name: Ensure go version
@@ -148,5 +149,9 @@ jobs:
         with:
           install_only: true
           version: v0.22.0
+          # Keep this until v1.31 (or superior) becomes the default kubectl version for the kind-action.
+          # It is used in podblocker shell script test to use --all-pods.
+          # If the podblocker e2e test relies on another way, this can also be removed.
+          kubectl_version: v1.31.0
       - name: Run specific e2e tests
         run: make e2e-test ARGS="-run ^${{ matrix.testname }}"

Makefile

@@ -51,7 +51,8 @@ dev-manifest:
 	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds.yaml > tests/kind/testfiles/kured-ds-concurrent-command.yaml
 	# concurrency e2e signal scenario
 	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds-signal.yaml > tests/kind/testfiles/kured-ds-concurrent-signal.yaml
-
+	# pod blocker e2e signal scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--blocking-pod-selector=name=temperamental/\1--blocking-pod-selector=app=blocker/g' kured-ds-signal.yaml > tests/kind/testfiles/kured-ds-podblocker.yaml
 
 e2e-test: dev-manifest dev-image
 	echo "Running ALL go tests"

tests/kind/main_test.go

@@ -384,3 +384,46 @@ func TestCordonningIsKept(t *testing.T) {
 		})
 	}
 }
+func TestE2EBlocker(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"podblocker",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := fmt.Sprintf(strconv.Itoa(rand.Intn(100)))
+			kindClusterName := fmt.Sprintf("kured-e2e-cordon-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-next.yaml")
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy(fmt.Sprintf("testfiles/kured-ds-%v.yaml", version)))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", fmt.Sprintf("testfiles/%v.sh",version), kindContext); err != nil {
+				t.Fatalf("node blocker test did not succeed: %v", err)
+			}
+		})
+	}
+}

tests/kind/testfiles/podblocker.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
+function gather_logs_and_cleanup {
+      for id in $(docker ps -q); do
+          echo "############################################################"
+          echo "docker logs for container $id:"
+          docker logs "$id"
+      done
+      ${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" logs ds/kured --all-pods -n kube-system
+}
+trap gather_logs_and_cleanup EXIT
+
+set +o errexit
+worker=$(${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o custom-columns=name:metadata.name --no-headers | grep worker | head -n 1)
+
+${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" label nodes "$worker" blocked-host=yes
+
+${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" apply -f - << EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+  labels:
+    app: blocker
+spec:
+  containers:
+    - name: nginx
+      image: nginx
+      imagePullPolicy: IfNotPresent
+  nodeSelector:
+    blocked-host: "yes"
+EOF
+
+docker exec "$worker" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+
+set -o errexit
+max_attempts="100"
+attempt_num=1
+sleep_time=5
+
+until ${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" logs ds/kured --all-pods -n kube-system  | grep -i -e "Reboot.*blocked"
+do
+  if (( attempt_num == max_attempts )); then
+      echo "Attempt $attempt_num failed and there are no more attempts left!"
+      exit 1
+  else
+      echo "Did not find 'reboot blocked' in the log, retrying in $sleep_time seconds (Attempt #$attempt_num)"
+      sleep "$sleep_time"
+  fi
+  (( attempt_num++ ))
+done