Skip to content

Commit

Permalink
Jenkins Pipeline Updates (NOAA-EMC#2815)
Browse files Browse the repository at this point in the history 
Pipeline updates:
- get `gh` location on remote machine and define global bash env
variable `$GH` for GitHub CLI
- Failed cases are now displayed accordingly in the Jenkins dashboard
(see NOTE below)
-  Added the Build # in messaging for clarity when running from re-runs.
- Replaced Matrix construct for concurrency with the parallel method
that can use dynamic case lists
- With removing of the hard coded list of cases we now get list of cases
dynamically from the PR case directory
- See new look of dashboard below (has more annotations and displays
only used cases)

NOTE: **failFast** (quitting all cases on failing of one) still does not
work because it isn't quitting the running remote shells. We can make
this a configurable capability in feature request with some custom code.
The current behavior has the remaining cases continuing to run after a
FAIL label has been issued and it is incumbent of the code manager to
kill the CI job in the controller before resetting another Ready label.

---------

Co-authored-by: David Huber <[email protected]>
  • Loading branch information
@TerrenceMcGuinness-NOAA @DavidHuber-NOAA @CoryMartin-NOAA
2 people authored and CoryMartin-NOAA committed Aug 13, 2024
1 parent 48b1cca commit 9280086
Showing 1 changed file with 9 additions and 6 deletions.
15 changes: 9 additions & 6 deletions ci/Jenkinsfile
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
def Machine = 'none'
def machine = 'none'
def CUSTOM_WORKSPACE = 'none'
def caseList = ''
def cases = ''
def GH = 'none'
// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
Expand Down Expand Up @@ -78,6 +79,7 @@ pipeline {
echo "Getting Common Workspace for ${Machine}"
ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim()
CUSTOM_WORKSPACE = "${WORKSPACE}"
sh(script: "mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS/*")
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
Expand All @@ -97,7 +99,7 @@ pipeline {
}
}
stages {
stage('build system') {
stage('Building') {
steps {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
script {
Expand All @@ -116,7 +118,7 @@ pipeline {
checkout scm
} catch (Exception e) {
if (env.CHANGE_ID) {
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine}: ${e.getMessage()}" """)
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine} in Build# ${env.BUILD_NUMBER}: ${e.getMessage()}" """)
}
STATUS = 'Failed'
error("Failed to checkout: ${e.getMessage()}")
Expand Down Expand Up @@ -149,7 +151,7 @@ pipeline {
try {
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
Expand All @@ -169,7 +171,7 @@ pipeline {
}
}
if (system == 'gfs') {
caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
}
}
}
Expand Down Expand Up @@ -276,6 +278,7 @@ pipeline {
}
}
}


stage( '5. FINALIZE' ) {
agent { label NodeName[machine].toLowerCase() }
Expand All @@ -291,7 +294,7 @@ pipeline {
""", returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
if (fileExists("${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log")) {
sh(script: """echo "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true)
sh(script: """echo "**CI ${STATUS}** on ${Machine} in Build# ${env.BUILD_NUMBER}<br>Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true)
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log """, returnStatus: true)
}
if (STATUS == 'Passed') {
Expand Down

0 comments on commit 9280086

Please sign in to comment.