[FSTORE-1702] Add base file for jenkins (#491)

* Add base file for jenkins * Trigger again * Try to fix the ci.Jenkinsfile * Test with agent * Blabla * Triiger * Fix the pipeline * apply some review * Trust * Trigger * Use yq * Trigger * Use filename extension * Fix inputs * Maybe numbers are strings * Change the host * Change the host * Automatic branch * Try to check if there are some cool env variables in jenkins * Weird * Crazy * Fix typo * Improve the jenkins ci * Fix inputs * Trigger ci * Trigger * try again * Fix inputs.json * Try the better way * Minor changes' * Wrap in step * Try to debug the pipeline * Remove ci jenkinsfile * Edit pipeline * Tests * Trigger again * Try the trigger * Missing bracket * Remove steps from post condition * Named argument * Remove the post condition to highlight the failure * Fix sh command * Fixed the wrong one * More is more * Printenv * Don't run post always * Remove post * Forgot to comment one step * Remove wrong quote * Redirect towards correct backend * Move to script and .github directory * Remove extra bracket * Consistent naming * Move cleanup * Try more declarative * Try more declarative * Fix json parsing * Try again * Wrong sha * Make it work * Var naming * Remove cleanup and declarative download * Post is weird * Query parameters test * Remove a quote * Add -G to apply query params * Query between times * Remove sha stuff * Sorted * Test * Test * Update CI * Missing tick * Why? * Test * Printenv * Check the loadtest PRs * Test again * Sleep 5 * Add workflow run_id when looking for a match * Test the loadtest PR * Try the workflow updated * Try and try * Minor cleanups * Remove head sha * cleanup inputs * Leave the PR on loadtest alone for now * Fix inputs * Use staging ip * Try triggering workflow from different branch * Fix labels inputs * Add junit parsing * Maybe 10s is a bit lwo * Remove extra * Remove results.xml system * Add loadtest PR back * Trigger * Set automatically same PR * Target correct endpoint * Concatenate labels if exist * Try again for labels * Test the labels * Fix * trigger again * Trigger again * Trigger happy * Trigger one more time * Move ci file + target correct ref branch + fix ref to main
logicalclocks · Mar 6, 2025 · 1fab3ee · 1fab3ee
1 parent fba6f04
commit 1fab3ee
Show file tree

Hide file tree

Showing 2 changed files with 158 additions and 0 deletions.
diff --git a/.github/ci.Jenkinsfile b/.github/ci.Jenkinsfile
@@ -0,0 +1,102 @@
+def WORKFLOW_RUN_ID = "0"
+def SHORT_SHA = ""
+def REF_LOADTEST_BRANCH = ""
+def WORKFLOW_RUN_URL = ""
+
+pipeline("E2E workflows") {
+  agent {
+    label "local"
+  }
+  environment {
+    GITHUB_TOKEN = credentials('990f5312-cd08-48ec-baf8-3b27ff551204')
+  }
+  stages {
+    stage('Clone repository') {
+      steps {
+        checkout scm
+      }
+    }
+    stage('Input parameters') {
+      steps {
+        script {
+          SHORT_SHA = sh(script: "git rev-parse --short HEAD", returnStdout: true).trim()
+          echo "Short sha: ${SHORT_SHA}"
+          sh "bash .github/workflow_inputs.sh ${SHORT_SHA}"
+          REF_LOADTEST_BRANCH = sh(script: "cat inputs.json | jq -r '.ref'", returnStdout: true).trim()
+          echo "Ref loadtest branch: ${REF_LOADTEST_BRANCH}"
+        }
+      }
+    }
+    stage('Post webhook') {
+      steps {
+        script {
+          def dispatch_response = sh(script: """curl -L -X POST -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -d @inputs.json \
+            https://api.github.com/repos/logicalclocks/loadtest/actions/workflows/e2e_small.yaml/dispatches""",
+            returnStdout: true
+          ).trim()
+          echo "Dispatch response: ${dispatch_response}"
+          sh "rm inputs.json"
+        }
+      }
+    }
+    stage ('Find workflow run id') {
+      steps {
+        script {
+          sleep 5
+          TIME_AFTER_WORKFLOW_DISPATCH = sh(script: "date -u +%Y-%m-%dT%H:%M:%SZ", returnStdout: true).trim()
+          WORKFLOW_RUN_ID = sh(script: """curl -L -X GET -G -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -d "event=workflow_dispatch" -d "actor=HopsworksJenkins" -d "branch=${REF_LOADTEST_BRANCH}" \
+            https://api.github.com/repos/logicalclocks/loadtest/actions/runs | jq -r '.workflow_runs[0].id'""", returnStdout: true).trim()
+          echo "Workflow run id: ${WORKFLOW_RUN_ID}"
+        }
+      }
+    }
+    stage('Wait for github action workflow to complete') {
+      steps {
+        script {
+          def status = "in_progress"
+          while (status == "in_progress" || status == "queued") {
+            sleep 60
+            status = sh(script: """curl -L -X GET -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              https://api.github.com/repos/logicalclocks/loadtest/actions/runs/${WORKFLOW_RUN_ID} | jq -r '.status' """, returnStdout: true).trim()
+            echo "Status: ${status}"
+          }
+        }
+      }
+    }
+    stage('Download artifacts') {
+      steps {
+        script {
+          def REPORT_URL = sh(
+            script: """curl -L -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              https://api.github.com/repos/logicalclocks/loadtest/actions/runs/${WORKFLOW_RUN_ID}/artifacts \
+              | jq -r '.artifacts[] | select(.name == "results_${WORKFLOW_RUN_ID}.xml") | .archive_download_url' """,
+            returnStdout: true
+          ).trim()
+          echo "Report url: ${REPORT_URL}"
+          sh(
+            script: """curl -L -H \"Accept: application/vnd.github+json\" \
+            -H \"Authorization: Bearer ${GITHUB_TOKEN}\" \
+            -H \"X-GitHub-Api-Version: 2022-11-28\" \
+            -o results.zip "${REPORT_URL}" """
+          )
+          sh """if [ -f results.xml ]; then rm results.xml; fi && unzip results.zip && rm results.zip"""
+        }
+      }
+    }
+  }
+  post {
+    always {
+      junit 'results.xml'
+    }
+  }
+}
diff --git a/.github/workflow_inputs.sh b/.github/workflow_inputs.sh
@@ -0,0 +1,56 @@
+#!/bin/bash 
+set -e
+
+SHORT_SHA=$1
+echo "" > inputs.yaml
+
+
+if [[ ${ghprbPullTitle} =~ (FSTORE-[0-9]+) || ${ghprbPullTitle} =~ (HWORKS-[0-9]+) ]]; then
+  captured_string=${BASH_REMATCH[1]}
+  echo "Found JIRA ticket: ${captured_string}, checking for corresponding pr in loadtest repo"
+  loadtest_prs=$(curl -L -G \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    -d "state=open" \
+    https://api.github.com/repos/logicalclocks/loadtest/pulls)
+
+  loadtest_branch=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .head.ref')
+  minikube_ip=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .labels[] | select(.name | contains("10.87.")) | .name')
+  labels=$(echo "${loadtest_prs}" | jq -r --arg captured_string ${captured_string} '.[] | select(.title | contains($captured_string)) | .labels[] | select(.name | contains("e2e")) | .name' | paste -sd ",")
+fi
+
+if [ -z "${loadtest_branch}" ]; then
+    echo "No corresponding pr found in loadtest repo, using main branch"
+    loadtest_branch="main"
+  else
+    echo "Found loadtest branch: ${loadtest_branch}"
+  fi
+
+  if [ -z "${minikube_ip}" ]; then
+    echo "No minikube ip found in labels, using default staging cluster" 
+    minikube_ip="stagingmain.devnet.hops.works" # Make it domain name instead of ip
+  else
+    echo "Found minikube ip in loadtest PR labels: ${minikube_ip}"
+  fi
+
+  if [ -z "${labels}" ]; then
+    echo "No labels found, using default e2e_small" 
+    labels="e2e_small"
+  else
+    echo "Found labels: ${labels}"
+  fi
+
+# .ref is the name of the branch where the workflow dispatch will be sent.
+yq '.ref = "main"' -i inputs.yaml
+
+yq '.inputs.max_parallel = "5"' -i inputs.yaml
+hopsworks_domain=$minikube_ip yq '.inputs.hopsworks_domain = strenv(hopsworks_domain)' -i inputs.yaml
+labels=$labels yq  '.inputs.labels = strenv(labels)' -i inputs.yaml
+hopsworks_api_branch=${ghprbSourceBranch} yq '.inputs.hopsworks_api_branch = strenv(hopsworks_api_branch)' -i inputs.yaml
+loadtest_branch=${loadtest_branch} yq '.inputs.loadtest_head_ref = strenv(loadtest_branch)' -i inputs.yaml
+short_sha=$SHORT_SHA yq '.inputs.short_sha = strenv(short_sha)' -i inputs.yaml
+user_repo_api=${ghprbPullAuthorLogin} yq '.inputs.user_repo_api = strenv(user_repo_api)' -i inputs.yaml
+
+yq -o=json inputs.yaml > inputs.json
+cat inputs.json