Merge pull request #689 from Roming22/ci-fix

Improve CI investigation
openshift-pipelines · Jun 23, 2023 · 1f6c7e7 · 1f6c7e7
2 parents c7fd18e + 7f2b977
commit 1f6c7e7
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 26 deletions.
diff --git a/.tekton/pipeline/acceptance-tests.yaml b/.tekton/pipeline/acceptance-tests.yaml
@@ -122,5 +122,7 @@ spec:
         - name: target_branch
           value: $(params.target_branch)
       workspaces:
+        - name: output
+          workspace: shared-workspace
         - name: source
           workspace: source
diff --git a/.tekton/pipeline/stonesoup-integeration-tests.yaml b/.tekton/pipeline/stonesoup-integeration-tests.yaml
@@ -87,5 +87,7 @@ spec:
         - name: region
           value: $(params.aws_region)
       workspaces:
+        - name: output
+          workspace: shared-workspace
         - name: source
           workspace: source
diff --git a/.tekton/pipeline/upgrade-tests.yaml b/.tekton/pipeline/upgrade-tests.yaml
@@ -26,7 +26,7 @@ spec:
         - name: url
           value: $(params.repo_url)
         - name: revision
-          value: $(params.target_branch)
+          value: $(params.revision)
     - name: generate-cluster-name
       taskRef:
         name: generate-cluster-name
@@ -63,11 +63,24 @@ spec:
           workspace: source
         - name: kubeconfig-dir
           workspace: upgrade-shared-workspace
-    - name: plnsvc-setup
+    - name: clone-pipeline-service-git-baseline
+      runAfter:
+        - "setup-ci-runner"
+      taskRef:
+        name: git-clone
+      workspaces:
+        - name: output
+          workspace: source
+      params:
+        - name: url
+          value: $(params.repo_url)
+        - name: revision
+          value: $(params.target_branch)
+    - name: plnsvc-setup-baseline
       taskRef:
         name: pipeline-service-setup
       runAfter:
-        - "setup-ci-runner"
+        - "clone-pipeline-service-git-baseline"
       workspaces:
         - name: kubeconfig-dir
           workspace: upgrade-shared-workspace
@@ -80,11 +93,11 @@ spec:
           value: $(params.revision)
         - name: target_branch
           value: $(params.target_branch)
-    - name: tests-before-upgrade
+    - name: tests-baseline
       taskRef:
         name: pipeline-service-tests
       runAfter:
-        - "plnsvc-setup"
+        - "plnsvc-setup-baseline"
       params:
         - name: target_branch
           value: $(params.target_branch)
@@ -93,9 +106,9 @@ spec:
           workspace: upgrade-shared-workspace
         - name: source
           workspace: source
-    - name: clone-pipeline-service-git-for-upgrade
+    - name: clone-pipeline-service-git-upgrade
       runAfter:
-        - "tests-before-upgrade"
+        - "tests-baseline"
       taskRef:
         name: git-clone
       workspaces:
@@ -106,11 +119,11 @@ spec:
           value: $(params.repo_url)
         - name: revision
           value: $(params.revision)
-    - name: plnsvc-upgrade-setup
+    - name: plnsvc-setup-upgrade
       taskRef:
         name: pipeline-service-upgrade-setup
       runAfter:
-        - "clone-pipeline-service-git-for-upgrade"
+        - "clone-pipeline-service-git-upgrade"
       workspaces:
         - name: kubeconfig-dir
           workspace: upgrade-shared-workspace
@@ -123,11 +136,11 @@ spec:
           value: $(params.revision)
         - name: target_branch
           value: $(params.target_branch)
-    - name: tests-post-upgrade
+    - name: tests-upgrade
       taskRef:
         name: pipeline-service-tests
       runAfter:
-        - "plnsvc-upgrade-setup"
+        - "plnsvc-setup-upgrade"
       params:
         - name: target_branch
           value: $(params.target_branch)
@@ -136,9 +149,9 @@ spec:
           workspace: upgrade-shared-workspace
         - name: source
           workspace: source
-    - name: clone-pipeline-service-git-post-upgrade
+    - name: clone-pipeline-service-git-downgrade
       runAfter:
-        - "tests-post-upgrade"
+        - "tests-upgrade"
       taskRef:
         name: git-clone
       workspaces:
@@ -149,11 +162,11 @@ spec:
           value: $(params.repo_url)
         - name: revision
           value: $(params.target_branch)
-    - name: plnsvc-downgrade-setup
+    - name: plnsvc-setup-downgrade
       taskRef:
         name: pipeline-service-upgrade-setup
       runAfter:
-        - "clone-pipeline-service-git-post-upgrade"
+        - "clone-pipeline-service-git-downgrade"
       workspaces:
         - name: kubeconfig-dir
           workspace: upgrade-shared-workspace
@@ -166,11 +179,11 @@ spec:
           value: $(params.revision)
         - name: target_branch
           value: $(params.target_branch)
-    - name: tests-post-downgrade
+    - name: tests-downgrade
       taskRef:
         name: pipeline-service-tests
       runAfter:
-        - "plnsvc-downgrade-setup"
+        - "plnsvc-setup-downgrade"
       params:
         - name: target_branch
           value: $(params.target_branch)
@@ -195,5 +208,7 @@ spec:
         - name: region
           value: $(params.aws_region)
       workspaces:
+        - name: output
+          workspace: upgrade-shared-workspace
         - name: source
           workspace: source
diff --git a/.tekton/tasks/destroy-cluster.yaml b/.tekton/tasks/destroy-cluster.yaml
@@ -11,6 +11,7 @@ spec:
     - name: target_branch
   workspaces:
     - name: source
+    - name: output
   steps:
     - name: destroy
       image: quay.io/redhat-pipeline-service/ci-runner:$(params.target_branch)
@@ -32,6 +33,8 @@ spec:
             secretKeyRef:
               name: hypershift-bitwarden
               key: "BW_PASSWORD"
+        - name: KUBECONFIG_DIR
+          value: "$(workspaces.output.path)"
         - name: REGION
           value: "$(params.region)"
       workingDir: "$(workspaces.source.path)"

diff --git a/ci/images/ci-runner/hack/bin/create-ci-runner-container.sh b/ci/images/ci-runner/hack/bin/create-ci-runner-container.sh
@@ -22,3 +22,10 @@ kubectl -n default apply -k "$MANIFEST_DIR/sidecar"
 
 kubectl -n default wait pod/ci-runner --for=condition=Ready --timeout=90s
 kubectl -n default describe pod/ci-runner
+
+echo
+echo "KUBECONFIG:"
+cat "$KUBECONFIG"
+echo
+echo "Connect to the ci-runner with: kubectl exec -n default --stdin --tty ci-runner -- bash"
+echo
diff --git a/ci/images/ci-runner/hack/bin/destroy-cluster.sh b/ci/images/ci-runner/hack/bin/destroy-cluster.sh
@@ -11,16 +11,21 @@ SCRIPT_DIR="$(
 
 # Give developers 15mins to connect to a pod and remove the file
 # if they want to investigate the failure
-if [ -e "$PWD/destroy-cluster.txt" ]; then
+failure_file="$PWD/destroy-cluster.txt"
+if [ -e "$failure_file" ]; then
+    echo "Failure detected."
+    echo "Delete '$failure_file' within 15 minutes to keep the cluster alive for investigation."
+    echo
+    echo "KUBECONFIG:"
+    cat "$KUBECONFIG_DIR/$KUBECONFIG"
+    echo
+    echo "Connect to the ci-runner with: kubectl exec -n default --stdin --tty ci-runner -- bash"
+    echo
+
     sleep 900
-    if [ -e "$PWD/destroy-cluster.txt" ]; then
+    if [ -e "$failure_file" ]; then
       echo "Failure is not being investigated, cluster will be destroyed."
     else
-      echo "KUBECONFIG:"
-      cat "$KUBECONFIG"
-      echo
-      echo "Connect to the ci-runner: kubectl exec -n default --stdin --tty ci-runner -- bash"
-      echo
       echo "Failure under investigation, cluster will not be destroyed."
       exit 1
     fi

diff --git a/ci/images/ci-runner/hack/sidecar/bin/plnsvc_test.sh b/ci/images/ci-runner/hack/sidecar/bin/plnsvc_test.sh
@@ -8,5 +8,9 @@ echo "Start executing pipeline cases ..."
 TEST_DIR=$(find "$PWD" -type f -name test.sh -exec dirname {} +)
 "$TEST_DIR/test.sh" --kubeconfig "$KUBECONFIG"
 
-# If the tests are successful, the cluster can be destroyed right away
-rm "$PWD/destroy-cluster.txt"
+
+# In case the user deleted the file early when they expected a failure
+if [ -e "$PWD/destroy-cluster.txt" ]; then
+    # If the tests are successful, the cluster can be destroyed right away
+    rm "$PWD/destroy-cluster.txt"
+fi