From b9d2bec52bf9eb1948c6c5a83aeda5c93b1dca43 Mon Sep 17 00:00:00 2001 From: Bruce Kropp - Raytheon <104453151+BruceKropp-Raytheon@users.noreply.github.com> Date: Wed, 22 Jan 2025 08:41:21 -0800 Subject: [PATCH] Feature epic metrics (#2536) * add CICD support to collect performace metrics during Build and Test stages Signed-off-by: Bruce Kropp * [AutoRT] Hera Job Completed. on-behalf-of @ufs-community * [AutoRT] Hercules Job Completed. on-behalf-of @ufs-community * [AutoRT] Derecho Job Completed. on-behalf-of @ufs-community * set logic to remove PR labels on success or failure Signed-off-by: Bruce Kropp * [AutoRT] Hera Job Completed. on-behalf-of @ufs-community * make sure Jet uses NAGAPE Signed-off-by: Bruce Kropp --------- Signed-off-by: Bruce Kropp Co-authored-by: epic-cicd-jenkins --- .cicd/Jenkinsfile | 210 +++++++++++++ .cicd/Jenkinsfile.combined | 388 ++++++++++++++++++++++++ .cicd/scripts/.shellcheckrc | 22 ++ .cicd/scripts/create_baseline.sh | 223 ++++++++++++++ .cicd/scripts/disk_usage.sh | 53 ++++ .cicd/scripts/post_test_results.sh | 75 +++++ .cicd/scripts/regression_test.sh | 169 +++++++++++ .cicd/scripts/wm_build.sh | 72 +++++ .cicd/scripts/wm_init.sh | 40 +++ .cicd/scripts/wm_test.sh | 143 +++++++++ tests/logs/RegressionTests_derecho.log | 4 +- tests/logs/RegressionTests_hera.log | 4 +- tests/logs/RegressionTests_hercules.log | 6 +- 13 files changed, 1403 insertions(+), 6 deletions(-) create mode 100644 .cicd/Jenkinsfile create mode 100644 .cicd/Jenkinsfile.combined create mode 100644 .cicd/scripts/.shellcheckrc create mode 100755 .cicd/scripts/create_baseline.sh create mode 100755 .cicd/scripts/disk_usage.sh create mode 100755 .cicd/scripts/post_test_results.sh create mode 100755 .cicd/scripts/regression_test.sh create mode 100755 .cicd/scripts/wm_build.sh create mode 100755 .cicd/scripts/wm_init.sh create mode 100755 .cicd/scripts/wm_test.sh diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile new file mode 100644 index 0000000000..d271ba32fb --- /dev/null +++ b/.cicd/Jenkinsfile @@ -0,0 +1,210 @@ +matchedNode = [] +generateBaselineNode = [] +for (label in pullRequest.labels) { + listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect { + node -> node.getLabelString().contains(label) ? node.name : null + + if ((label.matches(node.getLabelString()+"-(.*)"))) { + matchedNode += node.getLabelString() + } + + if ((label.matches(node.getLabelString()+"(.*)-BL"))) { + generateBaselineNode += node.getLabelString() + } + } +} + +modifiedLabels = matchedNode.collect{"'" + it + "'"} +baselineLabels = generateBaselineNode.collect{"'" + it + "'"} +def generateStage(nodeLabel) { + return { + stage("Initialize on ${nodeLabel.replaceAll("'","")}") { + node(nodeLabel) { + script { + currentBuild.displayName = "#${BUILD_NUMBER} ${nodeLabel.replaceAll("'","")} test=${params.WM_OPERATIONAL_TESTS}" + } + cleanWs() + checkout scm + script { + def UFS_PLATFORM = nodeLabel.replaceAll("'","") + echo "nodeLabel=${nodeLabel} NODE_NAME=${NODE_NAME} UFS_PLATFORM=${UFS_PLATFORM} UFS_COMPILER=${env.UFS_COMPILER}" + sh 'bash --login "${WORKSPACE}/.cicd/scripts/wm_init.sh"' + sh "STAGE_NAME='${env.STAGE_NAME}' " + 'bash --login "${WORKSPACE}/.cicd/scripts/disk_usage.sh"' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-time-wm_init.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-disk-usageInit*.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } + } + } + stage("Build on ${nodeLabel.replaceAll("'","")}") { + if (params.WM_BUILD == true ) { + node(nodeLabel) { + script { + def UFS_PLATFORM = nodeLabel.replaceAll("'","") + //currentBuild.displayName = "#${BUILD_NUMBER} ${nodeLabel.replaceAll("'","")} ${UFS_COMPILER}" + currentBuild.description = "build ${UFS_PLATFORM}-${UFS_COMPILER}" + + echo "Building on ${nodeLabel}" + sh 'bash --login "${WORKSPACE}/.cicd/scripts/wm_build.sh"' + sh "STAGE_NAME='${env.STAGE_NAME}' " + 'bash --login "${WORKSPACE}/.cicd/scripts/disk_usage.sh"' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-time-wm_build.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-disk-usageBuild.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } + } + } else { + echo "Building on ${nodeLabel} skipped" + } + } + stage("Test on ${nodeLabel.replaceAll("'","")}") { + if (params.WM_OPERATIONAL_TESTS != 'none' ) { + node(nodeLabel) { + script { + def UFS_PLATFORM = nodeLabel.replaceAll("'","") + try { + echo "Running Tests on ${nodeLabel}" + if (baselineLabels.contains(nodeLabel)) { + sh "WM_CREATE_BASELINE=true " + 'bash --login "${WORKSPACE}/.cicd/scripts/wm_test.sh"' + } + else { + sh "WM_CREATE_BASELINE=false " + 'bash --login "${WORKSPACE}/.cicd/scripts/wm_test.sh"' + } + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "wm_test_results-*-*.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } + catch(err) { + sh ''' + export machine=${NODE_NAME} + export CHANGE_ID=${CHANGE_ID} + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + set +x + + echo "Testing concluded...removing labels for $machine from $GIT_URL" + echo "https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels /{$machine-RT,$machine-BL}" + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/{$machine-RT,$machine-BL} + ''' + currentBuild.result = 'FAILURE' + } + + sh ''' + export machine=${NODE_NAME} + export CHANGE_ID=${CHANGE_ID} + cd ${WORKSPACE}/tests + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + + sh "STAGE_NAME='${env.STAGE_NAME}' " + 'bash --login "${WORKSPACE}/.cicd/scripts/disk_usage.sh"' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-time-wm_test.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-disk-usageTest.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + + sh "[[ -d tests/logs ]] && cd tests/logs && tar --create --gzip --verbose --dereference --file ../../wm_test_logs-${UFS_PLATFORM}-${env.UFS_COMPILER}.tgz log_${UFS_PLATFORM}/* RegressionTests_${UFS_PLATFORM}.log || cat /dev/null > ../../wm_test_logs-${UFS_PLATFORM}-${env.UFS_COMPILER}.tgz" + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "wm_test_logs-${UFS_PLATFORM}-${env.UFS_COMPILER}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } + } + } else { + echo "Running Tests on ${nodeLabel} skipped" + } + } + stage("Post from ${nodeLabel.replaceAll("'","")}") { + if (params.WM_OPERATIONAL_TESTS != 'none' ) { + node(nodeLabel) { + script { + def UFS_PLATFORM = nodeLabel.replaceAll("'","") + try { + echo "Post Results from ${nodeLabel}" + if (baselineLabels.contains(nodeLabel)) { + //sh 'bash --login "${WORKSPACE}/.cicd/scripts/post_test_results.sh ${NODE_NAME} BL"' + sh "WM_TEST_LABEL=BL " + 'bash --login "${WORKSPACE}/.cicd/scripts/post_test_results.sh"' + } + else { + sh "WM_TEST_LABEL=RT " + 'bash --login "${WORKSPACE}/.cicd/scripts/post_test_results.sh"' + } + } + catch(err) { + echo "Error: Post Results from ${nodeLabel}" + } + } + } + } + } + } +} + +def parallelStagesMap = modifiedLabels.collectEntries { + ["${it}" : generateStage(it)] +} + +pipeline { + agent none + parameters { + booleanParam name: 'WM_BUILD', defaultValue: false, description: 'Whether to attempt to compile the model code tests' + // Regression Test Suite ? + choice(name: 'WM_OPERATIONAL_TESTS', choices: ['default', 'control_p8', 'cpld_control_p8', 'comprehensive', 'rt.sh', 'none'], description: 'Specify the suite of tests to run') + } + environment { + ACCNR = 'epic' + AWS_PROD_ACCOUNT_ID = credentials('AWS_PROD_ACCOUNT_ID') + AWS_PROD_SNS_TOPIC = credentials('AWS_PROD_SNS_TOPIC') + GITHUB_TOKEN = credentials('GithubJenkinsNew') + GIT_URL = 'https://github.com/ufs-community/ufs-weather-model.git' + UFS_COMPILER = 'intel' + } + stages { + stage('Launch SonarQube') { + steps { + script { + echo "BRANCH_NAME=${env.CHANGE_BRANCH}" + echo "FORK_NAME=${env.CHANGE_FORK}" + echo "CHANGE_URL=${env.CHANGE_URL}" + echo "CHANGE_ID=${env.CHANGE_ID}" + build job: '/ufs-weather-model/ufs-wm-sonarqube', parameters: [ + string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'), + string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: ''), + string(name: 'CHANGE_URL', value: env.CHANGE_URL ?: ''), + string(name: 'CHANGE_ID', value: env.CHANGE_ID ?: '') + ], wait: false + } + } + } + stage('Run Regression Tests in Parallel') { + steps { + script { + parallel parallelStagesMap + } + } + } + } + post { + success { + node('built-in') { + echo 'This will run only if successful.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' + ''' + } + } + failure { + node('built-in') { + echo 'This will run only if the run was marked as unstable.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' + ''' + } + } + always { + script { + // Trigger another job to collect all build statistics + CI_JOB_NAME=env.JOB_NAME.replace("/${env.BRANCH_NAME}","") + CI_BRANCH_NAME=env.BRANCH_NAME.replace("%2F","%252F") + echo "post: Triggering ufs-weather-model/ufs-wm-metrics job for ${CI_JOB_NAME} on branch build ${CI_BRANCH_NAME}/${env.BUILD_NUMBER} ..." + + build job: '/ufs-weather-model/ufs-wm-metrics', parameters: [ + string(name: 'CI_JOB_NAME', value: "${CI_JOB_NAME}"), + string(name: 'CI_BUILD_NUMBER', value: "${CI_BRANCH_NAME}/${env.BUILD_NUMBER}") + ], wait: false + + echo "#### post: ufs-weather-model/ufs-wm-metrics COMPLETE." + } + } + } +} diff --git a/.cicd/Jenkinsfile.combined b/.cicd/Jenkinsfile.combined new file mode 100644 index 0000000000..b92bb07d2d --- /dev/null +++ b/.cicd/Jenkinsfile.combined @@ -0,0 +1,388 @@ +matchedNode = [] +generateBaselineNode = [] +for (label in pullRequest.labels) { + listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect { + node -> node.getLabelString().contains(label) ? node.name : null + + if ((label.matches(node.getLabelString()+"-(.*)"))) { + matchedNode += node.getLabelString() + } + + if ((label.matches(node.getLabelString()+"(.*)-BL"))) { + generateBaselineNode += node.getLabelString() + } + } +} + +modifiedLabels = matchedNode.collect{"'" + it + "'"} +baselineLabels = generateBaselineNode.collect{"'" + it + "'"} +def generateStage(nodeLabel) { + return { + stage("Running on ${nodeLabel}") { + node(nodeLabel) { + cleanWs() + checkout scm + script { + try { + echo "Running on ${nodeLabel}" + if (baselineLabels.contains(nodeLabel)) { + sh ''' + git submodule update --init --recursive + cd tests + pwd + export BL_DATE=$(cat bl_date.conf | cut -d '=' -f2) + export machine=${NODE_NAME} + export PATH=$PATH:~/bin + echo $CHANGE_ID + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + + if [[ $machine =~ "Jet" ]] + then + echo "Creating baselines on $machine" + export dprefix=/lfs1/NAGAPE/$ACCNR/$USER + ./rt.sh -a ${ACCNR} -c -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + elif [[ $machine =~ "Hercules" ]] + then + echo "Creating baselines on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/work/noaa/epic/hercules/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /work2/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + ./adjust_permissions.sh hercules develop-${BL_DATE} + chgrp noaa-hpc develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hercules.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hercules.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Orion" ]] + then + cd .. + #module load git/2.28.0 + git submodule update --init --recursive + cd tests + echo "Creating baselines on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed -i 's|/work/noaa/stmp/${USER}|/work/noaa/epic/stmp/role-epic/|g' rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/work/noaa/epic/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /work/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT/ + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + ./adjust_permissions.sh orion develop-${BL_DATE} + chgrp noaa-hpc develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_orion.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_orion.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Gaea" ]] + then + echo "Creating baselines on $machine" + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + unset LD_LIBRARY_PATH + export DISKNM=/gpfs/f5/epic/world-shared/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /gpfs/f5/epic/scratch/role.epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd ${DISKNM}/NEMSfv3gfs/ + chgrp ncep develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_gaea.log /gpfs/f5/epic/scratch/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_gaea.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Hera" ]] + then + echo "Creating baselines on $machine" + export ACCNR=epic + ./rt.sh -a ${ACCNR} -c -r -l rt.conf + export DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /scratch1/NCEPDEV/stmp4/role.epic/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hera.log /scratch2/NAGAPE/epic/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hera.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Derecho" ]] + then + echo "Creating baselines on $machine" + export ACCNR=nral0032 + ./rt.sh -a ${ACCNR} -c -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + export DISKNM=/glade/derecho/scratch/epicufsrt/ufs-weather-model/RT/ + cd ${DISKNM}/NEMSfv3gfs/ + mkdir develop-${BL_DATE} + cd /glade/derecho/scratch/epicufsrt/FV3_RT + rsync -a REGRESSION_TEST/ ${DISKNM}/NEMSfv3gfs/develop-${BL_DATE} + cd $WORKSPACE/tests + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_derecho.log /glade/derecho/scratch/epicufsrt/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_derecho.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + else + echo "Creating baselines on $machine" + ./rt.sh -a ${ACCNR} -c -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + fi + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + echo "Testing concluded...removing labels for $machine from $GIT_URL" + + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + git add logs/RegressionTests_$machine_name_logs.log + git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community " + git pull sshorigin $FORK_BRANCH + git push sshorigin HEAD:$FORK_BRANCH + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-BL + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + + } + else { + sh ''' + git submodule update --init --recursive + pwd + cd tests + export machine=${NODE_NAME} + export PATH=$PATH:~/bin + echo $CHANGE_ID + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + + if [[ $machine =~ "Jet" ]] + then + echo "Running regression tests on $machine" + export dprefix=/lfs1/NAGAPE/$ACCNR/$USER + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + elif [[ $machine =~ "Hercules" ]] + then + echo "Running regression tests on $machine" + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hercules.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hercules.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Orion" ]] + then + echo "Running regression tests on $machine" + cd .. + #module load git/2.28.0 + git submodule update --init --recursive + cd tests + export dprefix=/work2/noaa/$ACCNR/$USER + sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_orion.log /work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_orion.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Gaea" ]] + then + echo "Running regression tests on $machine" + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + unset LD_LIBRARY_PATH + cd logs/ + cp RegressionTests_gaea.log /gpfs/f5/epic/scratch/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_gaea.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Hera" ]] + then + echo "Running regression tests on $machine" + export ACCNR=epic + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_hera.log /scratch2/NAGAPE/epic/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_hera.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + elif [[ $machine =~ "Derecho" ]] + then + echo "Running regression tests on $machine" + export ACCNR=nral0032 + ./rt.sh -a ${ACCNR} -e -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + cd logs/ + cp RegressionTests_derecho.log /glade/derecho/scratch/epicufsrt/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + cp RegressionTests_derecho.log $WORKSPACE/tests/logs/ + cd $WORKSPACE/tests/ + else + echo "Running regression tests on $machine" + ./rt.sh -a ${ACCNR} -r -l rt.conf | tee $WORKSPACE/tests/logs/RT-run-$machine.log + fi + + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + echo "Testing concluded...removing labels for $machine from $GIT_URL" + git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + git add logs/RegressionTests_$machine_name_logs.log + git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community " + git pull sshorigin $FORK_BRANCH + git push sshorigin HEAD:$FORK_BRANCH + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-RT + + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } + } + catch(err) { + sh ''' + export machine=${NODE_NAME} + export CHANGE_ID=${CHANGE_ID} + export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url') + export FORK_BRANCH=$(curl --silent https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/$CHANGE_ID | jq -r '.head.ref') + cd $WORKSPACE/tests + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') + echo "Testing concluded...removing labels for $machine from $GIT_URL" + git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1 + + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log + + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/{$machine-RT,$machine-BL} + ''' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + currentBuild.result = 'FAILURE' + } + } + } + } + } +} + +def parallelStagesMap = modifiedLabels.collectEntries { + ["${it}" : generateStage(it)] +} + +pipeline { + agent none + environment { + ACCNR = 'epic' + AWS_PROD_ACCOUNT_ID = credentials('AWS_PROD_ACCOUNT_ID') + AWS_PROD_SNS_TOPIC = credentials('AWS_PROD_SNS_TOPIC') + GITHUB_TOKEN = credentials('GithubJenkinsNew') + GIT_URL = 'https://github.com/ufs-community/ufs-weather-model.git' + } + stages { + stage('Launch SonarQube') { + steps { + script { + echo "BRANCH_NAME=${env.CHANGE_BRANCH}" + echo "FORK_NAME=${env.CHANGE_FORK}" + echo "CHANGE_URL=${env.CHANGE_URL}" + echo "CHANGE_ID=${env.CHANGE_ID}" + build job: '/ufs-weather-model/ufs-wm-sonarqube', parameters: [ + string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'), + string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: ''), + string(name: 'CHANGE_URL', value: env.CHANGE_URL ?: ''), + string(name: 'CHANGE_ID', value: env.CHANGE_ID ?: '') + ], wait: false + } + } + } + stage('Run Regression Tests in Parallel') { + steps { + script { + parallel parallelStagesMap + } + } + } + } + post { + success { + node('built-in') { + echo 'This will run only if successful.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' + ''' + } + } + failure { + node('built-in') { + echo 'This will run only if the run was marked as unstable.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' + ''' + } + } + } +} diff --git a/.cicd/scripts/.shellcheckrc b/.cicd/scripts/.shellcheckrc new file mode 100644 index 0000000000..83d6166add --- /dev/null +++ b/.cicd/scripts/.shellcheckrc @@ -0,0 +1,22 @@ +# Shellcheck is used by Super-Linter to flag BASH shell oddities during CI pipeline. +# global settings are at top-level: ../../.shellcheckrc +# https://github.com/koalaman/shellcheck +# https://github.com/super-linter/super-linter + +# Allow to assign with local VAR=... +disable=SC2155 + +# Ignore source-able files from the varyious platforms in our matrix +disable=SC1091 + +# Disable info on pipe-ing commands +disable=SC2312 + +# Disable info sed with single quote string litteral +disable=SC2016 + +# Disable info on not double-quoting variables +disable=SC2086 + +# Disable info on &&, || chaining +disable=SC2015 diff --git a/.cicd/scripts/create_baseline.sh b/.cicd/scripts/create_baseline.sh new file mode 100755 index 0000000000..2ceed2f1fe --- /dev/null +++ b/.cicd/scripts/create_baseline.sh @@ -0,0 +1,223 @@ +#!/bin/bash -x + +export PATH=${PATH}:~/bin +echo "USER=${USER}" +echo "WORKSPACE=${WORKSPACE}" +export ACCNR=epic + +export account="-a ${ACCNR}" +export workflow="-e" + #[[ ${UFS_PLATFORM} = jet ]] && workflow="-r" + #[[ ${UFS_PLATFORM} = hera ]] && workflow="-r" + #[[ ${UFS_PLATFORM} =~ clusternoaa ]] && workflow="" + +export opt="-l" +export suite="rt.conf" + [[ -n ${WM_OPERATIONAL_TESTS} ]] && opt="-n" && suite="${WM_OPERATIONAL_TESTS} ${UFS_COMPILER}" || return 0 + [[ ${WM_OPERATIONAL_TESTS} = default ]] && opt="-n" && suite="control_p8 ${UFS_COMPILER}" + [[ ${WM_OPERATIONAL_TESTS} = comprehensive ]] && opt="-l" && suite="rt.conf" + [[ ${WM_OPERATIONAL_TESTS} = rt.conf ]] && opt="-l" && suite="rt.conf" + [[ "${suite}" = rt.conf ]] && opt="-l" + +set -eu + +export machine=${NODE_NAME} + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} + +pwd +ls -al .cicd/* +ls -al ${TESTS_DIR}/rt.sh + +function create_baseline() { + local machine=${1:-${NODE_NAME}} + local machine_id=${machine,,} # tolower + local WORKSPACE + WORKSPACE="$(pwd)" + local status=0 + + git submodule update --init --recursive + pwd + ls -al .cicd/* + cd tests + pwd + + [[ ${UFS_PLATFORM} =~ clusternoaa ]] && echo "export BL_DATE=20240426" > bl_date.conf || cat bl_date.conf + + mkdir -p logs/ + BL_DATE=$(cut -d '=' -f2 bl_date.conf) + export BL_DATE + + if [[ ${machine} =~ "Jet" ]] + then + echo "Creating baselines on ${machine}" + export dprefix=/lfs5/NAGAPE/${ACCNR}/${USER} + sed 's|/lfs4/HFIP/${ACCNR}/${USER}|/lfs4/HFIP/hfv3gfs/${USER}|g' -i rt.sh + sed 's|/lfs5/HFIP/${ACCNR}/${USER}|/lfs5/NAGAPE/${ACCNR}/${USER}|g' -i rt.sh + local workflow="-r" + ./rt.sh -a "${ACCNR}" -c "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + elif [[ ${machine} =~ "Hercules" ]] + then + echo "Creating baselines on ${machine}" + export dprefix=/work2/noaa/${ACCNR}/${USER} + sed "s|/noaa/stmp/|/noaa/${ACCNR}/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a "${ACCNR}" -c "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + export DISKNM=/work/noaa/epic/hercules/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir -p develop-${BL_DATE} + cd /work2/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT + ls -l REGRESSION_TEST/. + rsync -a --no-t REGRESSION_TEST/ "${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}" || echo "#### Warning! rsync $(pwd)/REGRESSION_TEST/ incomplete." + cd ${DISKNM}/NEMSfv3gfs/ + ./adjust_permissions.sh hercules "develop-${BL_DATE}" || : + chgrp noaa-hpc "develop-${BL_DATE}" || : + cd ${WORKSPACE}/tests + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + elif [[ ${machine} =~ "Orion" ]] + then + cd .. + #module load git/2.28.0 + git --version + git submodule update --init --recursive + cd tests + echo "Creating baselines on ${machine}" + export dprefix=/work2/noaa/${ACCNR}/${USER} + sed "s|/noaa/stmp/|/noaa/${ACCNR}/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a "${ACCNR}" -c "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + export DISKNM=/work/noaa/epic/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir -p develop-${BL_DATE} + cd /work/noaa/epic/stmp/role-epic/stmp/role-epic/FV3_RT/ + ls -l REGRESSION_TEST/. + rsync -a --no-t REGRESSION_TEST/ "${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}" || echo "#### Warning! rsync $(pwd)/REGRESSION_TEST/ incomplete." + cd ${DISKNM}/NEMSfv3gfs/ + ./adjust_permissions.sh orion "develop-${BL_DATE}" || : + chgrp noaa-hpc "develop-${BL_DATE}" || : + cd ${WORKSPACE}/tests + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + elif [[ ${machine} =~ "Gaea" ]] + then + echo "Creating baselines on ${machine}" + ./rt.sh -a "${ACCNR}" -c "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + unset LD_LIBRARY_PATH + export DISKNM=/gpfs/f5/epic/world-shared/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir -p develop-${BL_DATE} + cd /gpfs/f5/epic/scratch/role.epic/FV3_RT + ls -l REGRESSION_TEST/. + rsync -a --no-t REGRESSION_TEST/ "${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}" || echo "#### Warning! rsync $(pwd)/REGRESSION_TEST/ incomplete." + cd ${DISKNM}/NEMSfv3gfs/ + chgrp ncep "develop-${BL_DATE}" || : + cd ${WORKSPACE}/tests + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/gpfs/f5/epic/scratch/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + elif [[ ${machine} =~ "Hera" ]] + then + echo "Creating baselines on ${machine}" + export ACCNR=epic + sed "s|QUEUE=batch|QUEUE=windfall|g" -i rt.sh + local workflow="-r" + ./rt.sh -a "${ACCNR}" -c "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + export DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT + cd ${DISKNM}/NEMSfv3gfs/ + mkdir -p develop-${BL_DATE} + cd /scratch1/NCEPDEV/stmp4/role.epic/FV3_RT + ls -l REGRESSION_TEST/. + rsync -a --no-t REGRESSION_TEST/ "${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}" || echo "#### Warning! rsync $(pwd)/REGRESSION_TEST/ incomplete." + cd ${WORKSPACE}/tests + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/scratch2/NAGAPE/epic/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + elif [[ ${machine} =~ "Derecho" ]] + then + echo "Creating baselines on ${machine}" + export ACCNR=nral0032 + ./rt.sh -a "${ACCNR}" -c "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + export DISKNM=/glade/derecho/scratch/epicufsrt/ufs-weather-model/RT/ + cd ${DISKNM}/NEMSfv3gfs/ + mkdir -p develop-${BL_DATE} + cd /glade/derecho/scratch/epicufsrt/FV3_RT + ls -l REGRESSION_TEST/. + rsync -a --no-t REGRESSION_TEST/ "${DISKNM}/NEMSfv3gfs/develop-${BL_DATE}" || echo "#### Warning! rsync $(pwd)/REGRESSION_TEST/ incomplete." + cd ${WORKSPACE}/tests + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/glade/derecho/scratch/epicufsrt/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + else + echo "Creating baselines on ${machine}" + local workflow="-r" + ./rt.sh -a "${ACCNR}" -c "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + fi + + cd ${WORKSPACE} + + echo "Testing concluded for ${machine}. status=${status}" + return ${status} +} + +create_baseline "${machine}" diff --git a/.cicd/scripts/disk_usage.sh b/.cicd/scripts/disk_usage.sh new file mode 100755 index 0000000000..533897c8cb --- /dev/null +++ b/.cicd/scripts/disk_usage.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Output a CSV report of disk usage on subdirs of some path +# Usage: +# [JOB_NAME=] [BUILD_NUMBER=] [UFS_COMPILER=] [UFS_PLATFORM=] disk_usage path depth size outfile.csv +# +# args: +# directory=$1 +# depth=$2 +# size=$3 +# outfile=$4 + +export UFS_PLATFORM=${UFS_PLATFORM:-${NODE_NAME,,}} +export UFS_COMPILER=${UFS_COMPILER:-intel} +[[ -n ${WORKSPACE} ]] || WORKSPACE="$(pwd)" +[[ -n ${UFS_PLATFORM} ]] || UFS_PLATFORM="$(hostname -s 2>/dev/null)" || UFS_PLATFORM="$(hostname 2>/dev/null)" +[[ -n ${UFS_COMPILER} ]] || UFS_COMPILER="compiler" +echo "STAGE_NAME=${STAGE_NAME%% *}" # from pipeline + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +echo "script_dir=${script_dir}" + +# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set +# relative to script directory. +declare workspace +if [[ -d "${WORKSPACE}/${UFS_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${UFS_PLATFORM}" + outfile="${4:-${workspace}-${UFS_PLATFORM}-${UFS_COMPILER}-disk-usage${STAGE_NAME%% *}.csv}" +else + workspace="$(cd -- "${script_dir}/../.." && pwd)" + outfile="${4:-${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-disk-usage${STAGE_NAME%% *}.csv}" +fi +echo "workspace=${workspace}" +echo "outfile=${outfile}" + +function disk_usage() { + local directory="${1:-${PWD}}" + local depth="${2:-1}" + local size="${3:-k}" + echo "Disk usage: ${JOB_NAME:-ci}/${UFS_PLATFORM}/$(basename ${directory})" + ( + cd ${directory} || exit 1 + echo "Platform,Build,Owner,Group,Inodes,${size:-k}bytes,Access Time,Filename" + du -Px -d ${depth:-1} --inode --exclude='./workspace' | \ + while read -r line ; do + read -ra arr<<<"${line}"; inode="${arr[0]}"; filename="${arr[1]}"; + echo "${UFS_PLATFORM}-${UFS_COMPILER:-compiler},${JOB_NAME:-ci}/${BUILD_NUMBER:-0},$(stat -c '%U,%G' "${filename:-.}" || true),${inode:-0},$(du -Px -s -${size:-k} --time "${filename:-null}" 2>/dev/null || true)" | tr '\t' ',' || true; + done | sort -t, -k5 -n #-r + ) + echo "" +} + +disk_usage "${1}" "${2}" "${3}" | tee ${outfile} diff --git a/.cicd/scripts/post_test_results.sh b/.cicd/scripts/post_test_results.sh new file mode 100755 index 0000000000..26729ac123 --- /dev/null +++ b/.cicd/scripts/post_test_results.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# RT - RegressionTest label +# BL - Baseline label + +set -x +export machine=${1:-${NODE_NAME}} +label=${2:-${WM_TEST_LABEL}} +[[ -n "${label}" ]] || exit 1 + +export PATH=${PATH}:~/bin +echo "USER=${USER}" +echo "WORKSPACE=${WORKSPACE}" +export ACCNR=epic + +export account="-a ${ACCNR}" + +which jq + +set -eu + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} + +function post_test() { + local machine=${1:-${NODE_NAME}} + #local machine_id=${machine,,} # tolower + #local machine_name_logs=$(echo "${machine}" | awk '{ print tolower($1) }') + local label=${2:-"undef"} + local WORKSPACE + WORKSPACE="$(pwd)" + GIT_URL=${GIT_URL:-"ufs-weather-model"} + CHANGE_ID=${CHANGE_ID:-"develop"} + + GIT_OWNER=$(echo ${GIT_URL} | cut -d '/' -f4) + GIT_REPO_NAME=$(echo ${GIT_URL} | cut -d '/' -f5 | cut -d '.' -f1) + +set +x + echo "GIT_URL=${GIT_URL}" + echo "CHANGE_ID=${CHANGE_ID}" + echo "GIT_OWNER=${GIT_OWNER} GIT_REPO_NAME=${GIT_REPO_NAME}" + + echo "Testing concluded...removing label ${label} for ${machine} from ${GIT_URL}" + echo "https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels /${machine}-${label}" + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/${machine}-${label} +set -x + + git config user.email "ecc.platform@noaa.gov" + git config user.name "epic-cicd-jenkins" + + git add tests/logs/RegressionTests_${machine,,}.log + git status + git commit -m "[AutoRT] ${machine} Job Completed.\n\n\n on-behalf-of @ufs-community " + + SSH_ORIGIN=$(curl --silent "https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/${CHANGE_ID}" | jq -r '.head.repo.ssh_url') + git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1 + git remote add sshorigin ${SSH_ORIGIN} > /dev/null 2>&1 || return 0 + + FORK_BRANCH=$(curl --silent "https://api.github.com/repos/ufs-community/ufs-weather-model/pulls/${CHANGE_ID}" | jq -r '.head.ref') + git pull sshorigin ${FORK_BRANCH} || return 0 + git status + git push sshorigin HEAD:${FORK_BRANCH} || return 0 +} + +pwd +post_test "${machine}" "${label}" diff --git a/.cicd/scripts/regression_test.sh b/.cicd/scripts/regression_test.sh new file mode 100755 index 0000000000..4ed79eec40 --- /dev/null +++ b/.cicd/scripts/regression_test.sh @@ -0,0 +1,169 @@ +#!/bin/bash -x + +export PATH=${PATH}:~/bin +echo "USER=${USER}" +echo "WORKSPACE=${WORKSPACE}" +export ACCNR=epic + +export account="-a ${ACCNR}" +export workflow="-e" + #[[ ${UFS_PLATFORM} = jet ]] && workflow="-r" + #[[ ${UFS_PLATFORM} = hera ]] && workflow="-r" + #[[ ${UFS_PLATFORM} =~ clusternoaa ]] && workflow="" + +export opt="-l" +export suite="rt.conf" + [[ -n ${WM_OPERATIONAL_TESTS} ]] && opt="-n" && suite="${WM_OPERATIONAL_TESTS} ${UFS_COMPILER}" || return 0 + [[ ${WM_OPERATIONAL_TESTS} = default ]] && opt="-n" && suite="control_p8 ${UFS_COMPILER}" + [[ ${WM_OPERATIONAL_TESTS} = comprehensive ]] && opt="-l" && suite="rt.conf" + [[ ${WM_OPERATIONAL_TESTS} = rt.conf ]] && opt="-l" && suite="rt.conf" + [[ "${suite}" = rt.conf ]] && opt="-l" + +set -eu + +export machine=${NODE_NAME} + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} + +pwd +ls -al .cicd/* +ls -al ${TESTS_DIR}/rt.sh + +function regression_test() { + local machine=${1:-${NODE_NAME}} + local machine_id=${machine,,} # tolower + local WORKSPACE + WORKSPACE="$(pwd)" + local status=0 + + git submodule update --init --recursive + pwd + ls -al .cicd/* + cd tests + pwd + + [[ ${UFS_PLATFORM} =~ clusternoaa ]] && echo "export BL_DATE=20240426" > bl_date.conf || cat bl_date.conf + + mkdir -p logs/ + BL_DATE=$(cut -d '=' -f2 bl_date.conf) + export BL_DATE + + if [[ ${machine} =~ "Jet" ]] + then + echo "Running regression tests on ${machine}" + export dprefix=/lfs5/NAGAPE/${ACCNR}/${USER} + sed 's|/lfs4/HFIP/${ACCNR}/${USER}|/lfs4/HFIP/hfv3gfs/${USER}|g' -i rt.sh + sed 's|/lfs5/HFIP/${ACCNR}/${USER}|/lfs5/NAGAPE/${ACCNR}/${USER}|g' -i rt.sh + local workflow="-r" + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + elif [[ ${machine} =~ "Hercules" ]] + then + echo "Running regression tests on ${machine}" + export dprefix=/work2/noaa/${ACCNR}/${USER} + sed "s|/noaa/stmp/|/noaa/${ACCNR}/stmp/|g" -i rt.sh + export ACCNR=epic + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + elif [[ ${machine} =~ "Orion" ]] + then + echo "Running regression tests on ${machine}" + cd .. + #module load git/2.28.0 + git --version + git submodule update --init --recursive + cd tests + export dprefix=/work2/noaa/${ACCNR}/${USER} + sed "s|/noaa/stmp/|/noaa/${ACCNR}/stmp/|g" -i rt.sh + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/work/noaa/epic/role-epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + elif [[ ${machine} =~ "Gaea" ]] + then + echo "Running regression tests on ${machine}" + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + unset LD_LIBRARY_PATH + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/gpfs/f5/epic/scratch/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + elif [[ ${machine} =~ "Hera" ]] + then + echo "Running regression tests on ${machine}" + export ACCNR=epic + sed "s|QUEUE=batch|QUEUE=windfall|g" -i rt.sh + local workflow="-r" + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/scratch2/NAGAPE/epic/role.epic/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + elif [[ ${machine} =~ "Derecho" ]] + then + echo "Running regression tests on ${machine}" + export ACCNR=nral0032 + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + cd logs/ + cp "RegressionTests_${machine_id}.log" "$(dirname "${WORKSPACE}")" #/glade/derecho/scratch/epicufsrt/jenkins/workspace + git remote -v + git fetch --no-recurse-submodules origin + git reset FETCH_HEAD --hard + cd .. && cd .. && cd .. + pwd + cp "$(dirname "${WORKSPACE}")/RegressionTests_${machine_id}.log" "${WORKSPACE}/tests/logs/" + cd ${WORKSPACE}/tests/ + else + echo "Running regression tests on ${machine}" + local workflow="-r" + ./rt.sh -a "${ACCNR}" "${workflow}" "${opt}" "${suite}" | tee "${WORKSPACE}/tests/logs/RT-run-${machine}.log" + status=${PIPESTATUS[0]} + fi + + cd ${WORKSPACE} + + echo "Testing concluded for ${machine}. status=${status}" + return ${status} +} + +regression_test "${machine}" diff --git a/.cicd/scripts/wm_build.sh b/.cicd/scripts/wm_build.sh new file mode 100755 index 0000000000..e4a4c0822a --- /dev/null +++ b/.cicd/scripts/wm_build.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -eu +export UFS_PLATFORM=${UFS_PLATFORM:-${NODE_NAME,,}} +export UFS_COMPILER=${UFS_COMPILER:-intel} + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +cd "${UFS_MODEL_DIR}" +pwd +ls -l ./build.sh + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} +mkdir -p "${BUILD_DIR}" + +( + cd "${BUILD_DIR}" + pwd +) + +pwd +echo "NODE_NAME=${NODE_NAME}" +echo "UFS_PLATFORM=${UFS_PLATFORM}" +echo "UFS_COMPILER=${UFS_COMPILER}" +workspace=$(pwd) +export workspace +machine=${NODE_NAME} +echo "machine=<${machine}>" +machine_id=${UFS_PLATFORM,,} +if [[ ${UFS_PLATFORM} =~ clusternoaa ]] ; then + machine_id="noaacloud" + sed -e "s|EPIC/spack-stack/spack-stack-1.5.0|spack-stack/spack-stack-1.5.1|g" -i modulefiles/ufs_noaacloud.intel.lua +fi +echo "machine_id=<${machine_id}>" + +if [[ ${UFS_PLATFORM} = derecho ]] ; then + export ACCNR=nral0032 +else + export ACCNR=epic +fi +echo "ACCNR=${ACCNR}" + +export LMOD_SH_DBG_ON=0 +echo "LMOD_VERSION=${LMOD_VERSION}" +if [[ ${UFS_PLATFORM} = gaea ]] ; then + source /gpfs/f5/epic/scratch/role.epic/contrib/Lmod_init_C5.sh + echo "LMOD_VERSION=${LMOD_VERSION}" +fi +set +x +module use ${PWD}/modulefiles >/dev/null 2>&1 +module load ufs_${machine_id}.${UFS_COMPILER} || true +[[ ${UFS_PLATFORM} = gaea ]] && module load cmake/3.23.1 || true +module list + +echo "Pipeline Building WM on ${UFS_PLATFORM} ${UFS_COMPILER} with Account=${ACCNR}." +export CMAKE_FLAGS="-DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16" +/usr/bin/time -p \ + -o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-wm_build.json \ + -f '{\n "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \ + ./build.sh | tee ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-wm_build-log.txt +status=${PIPESTATUS[0]} +echo "Pipeline Completed WM build on ${UFS_PLATFORM} ${UFS_COMPILER}. status=${status}" + +ls -l build/ufs_model diff --git a/.cicd/scripts/wm_init.sh b/.cicd/scripts/wm_init.sh new file mode 100755 index 0000000000..2146b9e7ae --- /dev/null +++ b/.cicd/scripts/wm_init.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -eu +export UFS_PLATFORM=${UFS_PLATFORM:-${NODE_NAME,,}} +export UFS_COMPILER=${UFS_COMPILER:-intel} + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} + +cd "${UFS_MODEL_DIR}" +echo "UFS_PLATFORM=<${UFS_PLATFORM}>" +echo "UFS_COMPILER=<${UFS_COMPILER}>" + +pwd +echo "NODE_NAME=${NODE_NAME}" +echo "UFS_PLATFORM=${UFS_PLATFORM}" +echo "UFS_COMPILER=${UFS_COMPILER}" +workspace=$(pwd) +export workspace +machine=${NODE_NAME} +echo "machine=<${machine}>" +machine_id=${UFS_PLATFORM,,} +if [[ ${UFS_PLATFORM} =~ clusternoaa ]] ; then + machine_id="noaacloud" +fi +echo "machine_id=<${machine_id}>" + +/usr/bin/time -p \ + -o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-wm_init.json \ + -f '{\n "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \ + pwd diff --git a/.cicd/scripts/wm_test.sh b/.cicd/scripts/wm_test.sh new file mode 100755 index 0000000000..1377696730 --- /dev/null +++ b/.cicd/scripts/wm_test.sh @@ -0,0 +1,143 @@ +#!/bin/bash -x +set -eu +export UFS_PLATFORM=${UFS_PLATFORM:-${NODE_NAME,,}} +export UFS_COMPILER=${UFS_COMPILER:-intel} + +SCRIPT_REALPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPTS_DIR=$(dirname "${SCRIPT_REALPATH}") +UFS_MODEL_DIR=$(realpath "${SCRIPTS_DIR}/../..") +readonly UFS_MODEL_DIR +echo "UFS MODEL DIR: ${UFS_MODEL_DIR}" + +export CC=${CC:-mpicc} +export CXX=${CXX:-mpicxx} +export FC=${FC:-mpif90} + +BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} +TESTS_DIR=${TESTS_DIR:-${UFS_MODEL_DIR}/tests} + +( + cd "${TESTS_DIR}" + pwd + ls -al ./rt.sh +) + +export GIT_URL=${GIT_URL:-"ufs-weather-model"} +export CHANGE_ID=${CHANGE_ID:-"develop"} + +pwd +echo "GIT_URL=${GIT_URL}" +echo "CHANGE_ID=${CHANGE_ID}" +echo "NODE_NAME=${NODE_NAME}" +echo "USER=${USER}" +echo "UFS_PLATFORM=<${UFS_PLATFORM}>" +echo "UFS_COMPILER=<${UFS_COMPILER}>" +echo "WM_REGRESSION_TESTS=<${WM_REGRESSION_TESTS:-""}>" +echo "WM_OPERATIONAL_TESTS=<${WM_OPERATIONAL_TESTS:-""}>" +echo "WM_CREATE_BASELINE=<${WM_CREATE_BASELINE:-""}>" +echo "WM_POST_TEST_RESULTS=<${WM_POST_TEST_RESULTS:-""}>" + +machine=${NODE_NAME} +echo "machine=<${machine}>" +machine_id=${UFS_PLATFORM,,} +if [[ ${UFS_PLATFORM} =~ clusternoaa ]] ; then + machine_id="noaacloud" + #sed -i -e "s|EPIC/spack-stack/spack-stack-1.5.0|spack-stack/spack-stack-1.5.1|g" modulefiles/ufs_noaacloud.intel.lua +fi +echo "machine_id=<${machine_id}>" + +workspace=$(pwd) +export workspace + +status=0 + +ls -l build/ufs_model || : # just checking +status=$? + +[[ -n "${WM_REGRESSION_TESTS:-""}" ]] || WM_REGRESSION_TESTS=true # default +#[[ ${UFS_PLATFORM} == jet ]] && WM_REGRESSION_TESTS=false # takes too long +#[[ ${UFS_PLATFORM} == derecho ]] && WM_REGRESSION_TESTS=false +#[[ ${UFS_PLATFORM} =~ clusternoaa ]] && WM_REGRESSION_TESTS=false || : +export WM_REGRESSION_TESTS +[[ -n "${WM_CREATE_BASELINE:-""}" ]] || WM_CREATE_BASELINE=false # default +export WM_CREATE_BASELINE +[[ -n "${WM_POST_TEST_RESULTS:-""}" ]] || WM_POST_TEST_RESULTS=false # default +export WM_POST_TEST_RESULTS + +rm -f ${workspace}/wm_test_results-${UFS_PLATFORM}-${UFS_COMPILER}.txt + +if [[ ${WM_REGRESSION_TESTS} = true ]] ; then + echo "Pipeline Reqression Tests on ${UFS_PLATFORM} starting." + + export LMOD_SH_DBG_ON=0 + echo "LMOD_VERSION=${LMOD_VERSION}" + + set +x + if [[ ${UFS_PLATFORM} = orion ]] ; then + #module --ignore_cache load git/2.28.0 + git --version + git submodule update --init --recursive + fi + + if [[ ${UFS_PLATFORM} = gaea ]] ; then + source /gpfs/f5/epic/scratch/role.epic/contrib/Lmod_init_C5.sh + echo "LMOD_VERSION=${LMOD_VERSION}" + fi + + module use ${PWD}/modulefiles >/dev/null 2>&1 + module load ufs_${machine_id}.${UFS_COMPILER} || true + [[ ${UFS_PLATFORM} = gaea ]] && module load cmake/3.23.1 + module list + set -x + + echo "CHANGE_ID=${CHANGE_ID:-null}" + echo "ACCNR=${ACCNR}" + + [[ ! -f tests/logs/RegressionTests_${UFS_PLATFORM}.log ]] || mv tests/logs/RegressionTests_${UFS_PLATFORM}.log tests/logs/RegressionTests_${UFS_PLATFORM}.log.orig + + rm -f ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-wm_*-log.txt + umask 002 + if [[ ${WM_CREATE_BASELINE} = true ]] ; then + echo "start Creating baseline on ${UFS_PLATFORM} ..." + ls -al .cicd/* + echo "Pipeline Creating Baseline Tests ${WM_OPERATIONAL_TESTS:=default} on ${UFS_PLATFORM} ${UFS_COMPILER}" + /usr/bin/time -p \ + -o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-wm_test.json \ + -f '{\n "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \ + ./.cicd/scripts/create_baseline.sh | tee -a ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-wm_test-log.txt + status=${PIPESTATUS[0]} + echo "Pipeline Completed Baseline Tests ${WM_OPERATIONAL_TESTS} on ${UFS_PLATFORM} ${UFS_COMPILER}. status=${status}" + [[ ${WM_POST_TEST_RESULTS} = true ]] && ./.cicd/scripts/post_test_results.sh "${UFS_PLATFORM}" "BL" || echo "post test results seprately" + else + echo "skip Creating baseline on ${UFS_PLATFORM}." + ls -al .cicd/* + echo "Pipeline Running Regression Tests ${WM_OPERATIONAL_TESTS:=default} on ${UFS_PLATFORM} ${UFS_COMPILER}" + /usr/bin/time -p \ + -o ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-time-wm_test.json \ + -f '{\n "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' \ + ./.cicd/scripts/regression_test.sh | tee -a ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-wm_test-log.txt + status=${PIPESTATUS[0]} + echo "Pipeline Completed Regression Tests ${WM_OPERATIONAL_TESTS} on ${UFS_PLATFORM} ${UFS_COMPILER}. status=${status}" + [[ ${WM_POST_TEST_RESULTS} = true ]] && ./.cicd/scripts/post_test_results.sh "${UFS_PLATFORM}" "RT" || echo "post test results seprately" + fi + + cd tests/ + pwd + ls -al . + ## Check for log files ... + ls -al logs/. + + ## Test Results ... + echo "ExperimentName: ${WM_OPERATIONAL_TESTS:=default}" | tee -a ${workspace}/${UFS_PLATFORM}-${UFS_COMPILER}-wm_test-log.txt | tee ${workspace}/wm_test_results-${UFS_PLATFORM}-${UFS_COMPILER}.txt + grep -E " DIRECTORY: |Time: | Completed: |Result: " logs/RegressionTests_${UFS_PLATFORM}.log | tee -a ${workspace}/wm_test_results-${UFS_PLATFORM}-${UFS_COMPILER}.txt + grep -E " -- COMPILE | -- TEST " logs/RegressionTests_${UFS_PLATFORM}.log | tee -a ${workspace}/wm_test_results-${UFS_PLATFORM}-${UFS_COMPILER}.txt + + cd ${workspace} + find ${workspace}/tests/logs -ls + echo "Pipeline Reqression Tests on ${UFS_PLATFORM} complete. status=${status}" +else + echo "Pipeline Regression Tests on ${UFS_PLATFORM} (${machine}) skipped." + echo "ExperimentName: null" > ${workspace}/wm_test_results-${UFS_PLATFORM}-${UFS_COMPILER}.txt +fi + +exit ${status} diff --git a/tests/logs/RegressionTests_derecho.log b/tests/logs/RegressionTests_derecho.log index b639cca236..f058d47b54 100644 --- a/tests/logs/RegressionTests_derecho.log +++ b/tests/logs/RegressionTests_derecho.log @@ -6,10 +6,10 @@ UFSWM hash used in testing: Submodule hashes used in testing: 37cbb7d6840ae7515a9a8f0dfd4d89461b3396d1 AQM (v0.2.0-37-g37cbb7d) be5d28fd1b60522e6fc98aefeead20e6aac3530b AQM/src/model/CMAQ (CMAQv5.2.1_07Feb2018-198-gbe5d28fd1) - 2eccb9f74ebd8dfc09b53d15f5a96c847b77c443 CDEPS-interface/CDEPS (cdeps0.4.17-310-g2eccb9f) + 1f9eaaa142c8b07ed6b788c9f44ea02cc86d0bae CDEPS-interface/CDEPS (cdeps0.4.17-42-g1f9eaaa) ff7fd76118d47ccef6cb934b834161c6be5a9909 CICE-interface/CICE (CICE6.0.0-392-gff7fd76) 3792520561cf9419082ef41f9f0dffd03edf2e43 CICE-interface/CICE/icepack (Icepack1.1.0-198-g3792520) - b5d1cc189fced4abcb13fc70ed2febb2aef61757 CMEPS-interface/CMEPS (cmeps_v0.4.1-2313-gb5d1cc1) + 55576eba972ad53cc6546f00d409fa27361f78bd CMEPS-interface/CMEPS (cmeps_v0.4.1-2312-g55576eb) cabd7753ae17f7bfcc6dad56daf10868aa51c3f4 CMakeModules (v1.0.0-28-gcabd775) b40f99dd1701569ee27b01d53c3c8f32f4d33951 FV3 (heads/develop) 9490871a616604efe9dd75608214081eef557ad6 FV3/atmos_cubed_sphere (201912_public_release-408-g9490871) diff --git a/tests/logs/RegressionTests_hera.log b/tests/logs/RegressionTests_hera.log index ddd0451c6f..7ca73d83d6 100644 --- a/tests/logs/RegressionTests_hera.log +++ b/tests/logs/RegressionTests_hera.log @@ -5,9 +5,9 @@ UFSWM hash used in testing: Submodule hashes used in testing: 37cbb7d6840ae7515a9a8f0dfd4d89461b3396d1 AQM (v0.2.0-37-g37cbb7d) - 2eccb9f74ebd8dfc09b53d15f5a96c847b77c443 CDEPS-interface/CDEPS (cdeps0.4.17-310-g2eccb9f) + 1f9eaaa142c8b07ed6b788c9f44ea02cc86d0bae CDEPS-interface/CDEPS (cdeps0.4.17-42-g1f9eaaa) ff7fd76118d47ccef6cb934b834161c6be5a9909 CICE-interface/CICE (CICE6.0.0-392-gff7fd76) - b5d1cc189fced4abcb13fc70ed2febb2aef61757 CMEPS-interface/CMEPS (cmeps_v0.4.1-2313-gb5d1cc1) + 55576eba972ad53cc6546f00d409fa27361f78bd CMEPS-interface/CMEPS (cmeps_v0.4.1-2312-g55576eb) cabd7753ae17f7bfcc6dad56daf10868aa51c3f4 CMakeModules (v1.0.0-28-gcabd775) b40f99dd1701569ee27b01d53c3c8f32f4d33951 FV3 (heads/develop) 041422934cae1570f2f0e67239d5d89f11c6e1b7 GOCART (sdr_v2.1.2.6-119-g0414229) diff --git a/tests/logs/RegressionTests_hercules.log b/tests/logs/RegressionTests_hercules.log index 28ad792532..589154b88c 100644 --- a/tests/logs/RegressionTests_hercules.log +++ b/tests/logs/RegressionTests_hercules.log @@ -1,15 +1,16 @@ ====START OF hercules REGRESSION TESTING LOG==== UFSWM hash used in testing: + 8b8eac654f051017e26b76099b5eb7471f0a110d Submodule hashes used in testing: 37cbb7d6840ae7515a9a8f0dfd4d89461b3396d1 AQM (v0.2.0-37-g37cbb7d) be5d28fd1b60522e6fc98aefeead20e6aac3530b AQM/src/model/CMAQ (CMAQv5.2.1_07Feb2018-198-gbe5d28fd1) - 2eccb9f74ebd8dfc09b53d15f5a96c847b77c443 CDEPS-interface/CDEPS (cdeps0.4.17-310-g2eccb9f) + 1f9eaaa142c8b07ed6b788c9f44ea02cc86d0bae CDEPS-interface/CDEPS (cdeps0.4.17-42-g1f9eaaa) ff7fd76118d47ccef6cb934b834161c6be5a9909 CICE-interface/CICE (CICE6.0.0-392-gff7fd76) 3792520561cf9419082ef41f9f0dffd03edf2e43 CICE-interface/CICE/icepack (Icepack1.1.0-198-g3792520) - b5d1cc189fced4abcb13fc70ed2febb2aef61757 CMEPS-interface/CMEPS (cmeps_v0.4.1-2313-gb5d1cc1) + 55576eba972ad53cc6546f00d409fa27361f78bd CMEPS-interface/CMEPS (cmeps_v0.4.1-2312-g55576eb) cabd7753ae17f7bfcc6dad56daf10868aa51c3f4 CMakeModules (v1.0.0-28-gcabd775) b40f99dd1701569ee27b01d53c3c8f32f4d33951 FV3 (heads/develop) 9490871a616604efe9dd75608214081eef557ad6 FV3/atmos_cubed_sphere (201912_public_release-408-g9490871) @@ -37,6 +38,7 @@ The first time is for the full script (prep+run+finalize). The second time is specifically for the run phase. Times/Memory will be empty for failed tests. + BASELINE DIRECTORY: /work/noaa/epic/hercules/UFS-WM_RT/NEMSfv3gfs/develop-20250107 COMPARISON DIRECTORY: /work2/noaa/stmp/jongkim/jongkim/FV3_RT/rt_2292685