From a1b8d1f9dcdef8182735c8e7bca3f1eb8a3646a1 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Thu, 15 Feb 2024 15:35:14 -0700 Subject: [PATCH 01/30] test --- test | 1 + 1 file changed, 1 insertion(+) create mode 100644 test diff --git a/test b/test new file mode 100644 index 0000000..002d9aa --- /dev/null +++ b/test @@ -0,0 +1 @@ +pedantic comment From 75c2ff84e3c3008fb9758b1f4f93be1dec8da2b0 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 09:08:48 -0700 Subject: [PATCH 02/30] platform-1077, adding initial CI/CD for Jenkins --- .cicd/Jenkinsfile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .cicd/Jenkinsfile diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile new file mode 100644 index 0000000..79a500f --- /dev/null +++ b/.cicd/Jenkinsfile @@ -0,0 +1,40 @@ +pipeline { + agent none + stages { + stage('Run Unit Tests') { + agent { + label 'hera' + } + environment { + } + steps { + script { + cleanWs() + checkout scm + sh ''' + echo "Running on ${NODE_NAME}" + ''' + } + } + } + + post { + success { + node('built-in') { + echo 'This will run only if successful.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' + ''' + } + } + failure { + node('built-in') { + echo 'This will run only if the run was marked as unstable.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' + ''' + } + } + } + } +} From a6ba2a06f7c255303a5a2ee06fd0243a461bb50b Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 09:17:11 -0700 Subject: [PATCH 03/30] Removing typo. --- .cicd/Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 79a500f..56405e2 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -5,8 +5,6 @@ pipeline { agent { label 'hera' } - environment { - } steps { script { cleanWs() From ac9750c20ef54c14f13c772bc18fc2baab20ab1c Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 09:23:22 -0700 Subject: [PATCH 04/30] Fixing other typo --- .cicd/Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 56405e2..cb3bedb 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -15,7 +15,7 @@ pipeline { } } } - + } post { success { node('built-in') { @@ -34,5 +34,4 @@ pipeline { } } } - } } From 22bcb97167d7505ad764ce800f9e3b978bf7b92c Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 09:29:40 -0700 Subject: [PATCH 05/30] Adding unit tests script --- .cicd/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index cb3bedb..4e81993 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -10,7 +10,9 @@ pipeline { cleanWs() checkout scm sh ''' - echo "Running on ${NODE_NAME}" + echo "Running Unit Tests on ${NODE_NAME}" + cd unit_tests + ./run_unit_tests.sh ''' } } From 5705b9bb40150ae68d6e2ec1bcb639f2281fdfc3 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:02:04 -0700 Subject: [PATCH 06/30] Adding label removal --- .cicd/Jenkinsfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 4e81993..80d9844 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -10,9 +10,8 @@ pipeline { cleanWs() checkout scm sh ''' - echo "Running Unit Tests on ${NODE_NAME}" cd unit_tests - ./run_unit_tests.sh + echo "Running Unit Tests on ${NODE_NAME}" ''' } } @@ -23,6 +22,10 @@ pipeline { node('built-in') { echo 'This will run only if successful.' sh ''' + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + echo "Testing successfully completed...removing label(s)" + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}' aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' ''' } From 0cf35af16ca662bf7d68f374415dfbb0161d5256 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:06:24 -0700 Subject: [PATCH 07/30] Adding auth token from Jenkins. --- .cicd/Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 80d9844..a59b9aa 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -1,5 +1,8 @@ pipeline { agent none + environment { + GITHUB_TOKEN = credentials('GithubJenkinsNew') + } stages { stage('Run Unit Tests') { agent { From 697fdbac99530d6a6eec9679f583d8f6ad74a8dd Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:21:02 -0700 Subject: [PATCH 08/30] Checking env variables --- .cicd/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index a59b9aa..55b534c 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -13,6 +13,7 @@ pipeline { cleanWs() checkout scm sh ''' + env cd unit_tests echo "Running Unit Tests on ${NODE_NAME}" ''' From 2831d28d1f2491392e065afc126dbe0ecb7af6cb Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:27:00 -0700 Subject: [PATCH 09/30] Troubleshooting --- .cicd/Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 55b534c..9c18448 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -26,8 +26,9 @@ pipeline { node('built-in') { echo 'This will run only if successful.' sh ''' - GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) - GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + echo $GIT_URL + export GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + export GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) echo "Testing successfully completed...removing label(s)" curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}' aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' From ba0d045fa5f6c369ebfe0361253f6d85d39fe898 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:29:03 -0700 Subject: [PATCH 10/30] More troubleshooting --- .cicd/Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 9c18448..aa2ec94 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -2,6 +2,7 @@ pipeline { agent none environment { GITHUB_TOKEN = credentials('GithubJenkinsNew') + GIT_URL = $GIT_URL } stages { stage('Run Unit Tests') { @@ -27,8 +28,8 @@ pipeline { echo 'This will run only if successful.' sh ''' echo $GIT_URL - export GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) - export GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) echo "Testing successfully completed...removing label(s)" curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}' aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' From fd3b29ad02f606ab6cbb611887769967ca8e3a8b Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:30:22 -0700 Subject: [PATCH 11/30] Updating --- .cicd/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index aa2ec94..e00a744 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent none environment { GITHUB_TOKEN = credentials('GithubJenkinsNew') - GIT_URL = $GIT_URL + GIT_URL = ${GIT_URL} } stages { stage('Run Unit Tests') { From 3d62c00f6cd86ab5911ecfb9e16c3b8b37dc84b9 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:31:59 -0700 Subject: [PATCH 12/30] Updating --- .cicd/Jenkinsfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index e00a744..d1a9308 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -2,7 +2,6 @@ pipeline { agent none environment { GITHUB_TOKEN = credentials('GithubJenkinsNew') - GIT_URL = ${GIT_URL} } stages { stage('Run Unit Tests') { @@ -27,9 +26,9 @@ pipeline { node('built-in') { echo 'This will run only if successful.' sh ''' - echo $GIT_URL - GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) - GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + echo ${env.GIT_URL} + GIT_OWNER=$(echo ${env.GIT_URL} | cut -d '/' -f4) + GIT_REPO_NAME=$(echo ${env.GIT_URL} | cut -d '/' -f5 | cut -d '.' -f1) echo "Testing successfully completed...removing label(s)" curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}' aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' From 43f20e1621abb6880d50e331d134f0f7238d4a26 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:34:39 -0700 Subject: [PATCH 13/30] Updating --- .cicd/Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index d1a9308..d44d062 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -13,7 +13,6 @@ pipeline { cleanWs() checkout scm sh ''' - env cd unit_tests echo "Running Unit Tests on ${NODE_NAME}" ''' @@ -26,9 +25,10 @@ pipeline { node('built-in') { echo 'This will run only if successful.' sh ''' - echo ${env.GIT_URL} - GIT_OWNER=$(echo ${env.GIT_URL} | cut -d '/' -f4) - GIT_REPO_NAME=$(echo ${env.GIT_URL} | cut -d '/' -f5 | cut -d '.' -f1) + env + echo $GIT_URL + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) echo "Testing successfully completed...removing label(s)" curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}' aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' From 967efff8a0b1f65cb00ffc00db199004944ea86e Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:39:24 -0700 Subject: [PATCH 14/30] Updating to use CHANGE_URL rather than GIT_URL --- .cicd/Jenkinsfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index d44d062..40f933a 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -25,10 +25,8 @@ pipeline { node('built-in') { echo 'This will run only if successful.' sh ''' - env - echo $GIT_URL - GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) - GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) + GIT_OWNER=$(echo $CHANGE_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $CHANGE_URL | cut -d '/' -f5 | cut -d '.' -f1) echo "Testing successfully completed...removing label(s)" curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}' aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' From 00d128911205f21ed3f78841fa93df9abc9d252f Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:49:41 -0700 Subject: [PATCH 15/30] Updating to use token for testing repo. --- .cicd/Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 40f933a..4205387 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -1,7 +1,8 @@ pipeline { agent none environment { - GITHUB_TOKEN = credentials('GithubJenkinsNew') +// GITHUB_TOKEN = credentials('GithubJenkinsNew') + GITHUB_TOKEN = credentials('kb-github-test-token') } stages { stage('Run Unit Tests') { From da3280cc451e4f35b5e111d7d2b411e031b987ab Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 10:58:53 -0700 Subject: [PATCH 16/30] Updating --- .cicd/Jenkinsfile | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 4205387..7c77489 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -3,6 +3,7 @@ pipeline { environment { // GITHUB_TOKEN = credentials('GithubJenkinsNew') GITHUB_TOKEN = credentials('kb-github-test-token') + AWS_ACCOUNT_ID = 211527314271 } stages { stage('Run Unit Tests') { @@ -22,15 +23,21 @@ pipeline { } } post { - success { + always { node('built-in') { - echo 'This will run only if successful.' sh ''' GIT_OWNER=$(echo $CHANGE_URL | cut -d '/' -f4) GIT_REPO_NAME=$(echo $CHANGE_URL | cut -d '/' -f5 | cut -d '.' -f1) echo "Testing successfully completed...removing label(s)" curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}' - aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' + ''' + } + } + success { + node('built-in') { + echo 'This will run only if successful.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_ACCOUNT_ID}:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' ''' } } @@ -38,7 +45,7 @@ pipeline { node('built-in') { echo 'This will run only if the run was marked as unstable.' sh ''' - aws sns publish --topic-arn "arn:aws:sns:us-east-1:211527314271:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_ACCOUNT_ID}:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' ''' } } From bef267ea5054396d70fb4f93a86d081bc0239853 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Tue, 20 Feb 2024 11:18:46 -0700 Subject: [PATCH 17/30] Updating auth --- .cicd/Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 7c77489..905326b 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -1,8 +1,7 @@ pipeline { agent none environment { -// GITHUB_TOKEN = credentials('GithubJenkinsNew') - GITHUB_TOKEN = credentials('kb-github-test-token') + GITHUB_TOKEN = credentials('GithubJenkinsNew') AWS_ACCOUNT_ID = 211527314271 } stages { From 4e7af4c1561733b0c0906a4ac60eac9ba64506ee Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Thu, 22 Feb 2024 10:37:12 -0700 Subject: [PATCH 18/30] PLATFORM-1077, removing pendatic comment file. --- test | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test diff --git a/test b/test deleted file mode 100644 index 002d9aa..0000000 --- a/test +++ /dev/null @@ -1 +0,0 @@ -pedantic comment From 9c95c32cbc390c27dd9717eb48d575352ea85397 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 12:08:28 -0400 Subject: [PATCH 19/30] Update Jenkinsfile --- .cicd/Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 905326b..056d3e9 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -16,6 +16,9 @@ pipeline { sh ''' cd unit_tests echo "Running Unit Tests on ${NODE_NAME}" + sbatch run_standalone.sh + cd stochy_out + ls ''' } } From 35c32c01053c2b3095c9c9f3a2f20af71d646d7f Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 12:10:45 -0400 Subject: [PATCH 20/30] Update stochastic_physics.F90 fix do_sppt logic --- stochastic_physics.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stochastic_physics.F90 b/stochastic_physics.F90 index 0f1699f..0dcd831 100644 --- a/stochastic_physics.F90 +++ b/stochastic_physics.F90 @@ -164,7 +164,7 @@ subroutine init_stochastic_physics(levs, blksz, dtp, sppt_amp, input_nml_file_in endif enddo if (sppt_sfclimit) then - do k=1,7 + do k=1,levs vfact_sppt(k)=pbl_taper(k) enddo endif From 9f235a457a7f31b42cd29306a9da546eb770ef97 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 12:12:03 -0400 Subject: [PATCH 21/30] Create input.nml --- unit_tests/input.nml | 89 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 unit_tests/input.nml diff --git a/unit_tests/input.nml b/unit_tests/input.nml new file mode 100644 index 0000000..723a888 --- /dev/null +++ b/unit_tests/input.nml @@ -0,0 +1,89 @@ +&amip_interp_nml + interp_oi_sst = .true. + use_ncep_sst = .true. + use_ncep_ice = .false. + no_anom_sst = .false. + data_set = 'reynolds_oi', + date_out_of_range = 'climo', +/ + +&atmos_model_nml + blocksize = 32 + chksum_debug = .false. + dycore_only = .false. + fhout = 1 + fhmax = 800 + fhmaxhf = -1 + ccpp_suite = 'FV3_GFS_v15p2_coupled' +/ + +&diag_manager_nml + prepend_date = .F. +/ + +&fms_io_nml + checksum_required = .false. + max_files_r = 100, + max_files_w = 100, +/ + +&fms_nml + clock_grain = 'ROUTINE', + domains_stack_size = 40000000, + print_memory_usage = .false. +/ + +&fv_grid_nml + grid_file = 'INPUT/grid_spec.nc' +/ + +&fv_core_nml + layout = 1,4 + io_layout = 1,1 + npx = 97, + npy = 97, + ntiles = 6, + grid_type = -1 + +/ + +&gfs_physics_nml + do_sppt = .true. + do_shum = .true. + do_skeb = .true. + lndp_type = 2 + n_var_lndp = 2 +/ + +&nam_sppperts + +/ +&nam_stochy + SKEBNORM=1, + SKEB_NPASS=30, + SKEB_VDOF=5, + SKEB=0.5, + SKEB_TAU=2.16E4, + SKEB_LSCALE=1000.E3, + SHUM=0.01, + SHUM_TAU=21600, + SHUM_LSCALE=500000, + SPPT=0.8,0.4,0.1 + SPPT_TAU=21600,86400,216000 + SPPT_LSCALE=500000,1000000,2000000 + SPPT_LOGIT=.TRUE., + SPPT_SFCLIMIT=.TRUE., + ISEED_SHUM=1, + ISEED_SKEB=2, + ISEED_SPPT=3, + STOCHINI=.false. +/ +&nam_sfcperts + lndp_type=2 + lndp_var_list='vgf','smc' + lndp_prt_list=1.0,0.5 + iseed_lndp=4,5 + lndp_tau= 86400, + lndp_lscale = 250000, + lndp_model_type = 1 +/ From 0de7a90cff3c2a9e6c958cbafa823c2c9888e7b1 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 12:13:58 -0400 Subject: [PATCH 22/30] Update run_standalone.sh input-data and account --- unit_tests/run_standalone.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_tests/run_standalone.sh b/unit_tests/run_standalone.sh index 2ec2858..bb81516 100755 --- a/unit_tests/run_standalone.sh +++ b/unit_tests/run_standalone.sh @@ -1,7 +1,7 @@ #!/bin/sh #SBATCH -e err #SBATCH -o out -#SBATCH --account=gsienkf +#SBATCH --account=epic #SBATCH --qos=debug #SBATCH --nodes=1 #SBATCH --ntasks-per-node=40 @@ -25,7 +25,7 @@ fi # copy input directory if [ ! -d INPUT ]; then - cp -r /scratch2/BMC/gsienkf/Philip.Pegion/stochastic_physics_unit_tests/input_data INPUT + cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT fi mkdir -p RESTART @@ -40,6 +40,6 @@ sed -i -e "s/RES/$RES/g" input.nml sed -i -e "s/_STOCHINI_/.false./g" input.nml export OMP_NUM_THREADS=2 module list -time srun --label -n 24 standalone_stochy.x +time srun --label -n 24 standalone_stochy.x mkdir stochy_out mv workg* stochy_out From ebb4758e2659ea1b29a0af404e961281704ea114 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 12:16:42 -0400 Subject: [PATCH 23/30] Update standalone_stochy.F90 add new spp parameters and allocation --- unit_tests/standalone_stochy.F90 | 35 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/unit_tests/standalone_stochy.F90 b/unit_tests/standalone_stochy.F90 index d91e368..aec584c 100644 --- a/unit_tests/standalone_stochy.F90 +++ b/unit_tests/standalone_stochy.F90 @@ -1,5 +1,5 @@ program standalone_stochy - +use mpi_f08 use stochastic_physics, only : init_stochastic_physics,run_stochastic_physics use get_stochy_pattern_mod, only : write_stoch_restart_atm @@ -16,6 +16,7 @@ program standalone_stochy implicit none integer, parameter :: nlevs=3 integer, parameter :: max_n_var_lndp = 6 +integer, parameter :: max_n_var_spp = 6 integer :: ntasks,fid integer :: nthreads integer :: ncid,xt_dim_id,yt_dim_id,time_dim_id,xt_var_id,yt_var_id,time_var_id,var_id_lat,var_id_lon,var_id_tile @@ -24,9 +25,11 @@ program standalone_stochy integer :: zt_dim_id,zt_var_id character*2 :: strid -character(len=3), dimension(max_n_var_lndp) :: lndp_var_list -real(kind=kind_dbl_prec), dimension(max_n_var_lndp) :: lndp_prt_list -include 'mpif.h' +character(len=3), allocatable :: lndp_var_list(:) +real(kind=kind_dbl_prec), allocatable :: lndp_prt_list(:) +character(len=10), allocatable :: spp_var_list(:) +real(kind=kind_dbl_prec), allocatable :: spp_prt_list(:) +real(kind=kind_dbl_prec), allocatable :: spp_stddev_cutoff(:) include 'netcdf.inc' real :: ak(nlevs+1),bk(nlevs+1) real(kind=4) :: ts,undef @@ -42,7 +45,7 @@ program standalone_stochy integer :: i1,i2,j1,npts,istart,tpt character*80 :: fname character*1 :: ntile_out_str -integer :: comm +type(MPI_Comm) :: comm real(kind=4),allocatable,dimension(:,:) :: workg,tile_number real(kind=4),allocatable,dimension(:,:,:) :: workg3d @@ -58,18 +61,19 @@ program standalone_stochy real (kind=kind_phys),allocatable :: skebu_wts (:,:,:) real (kind=kind_phys),allocatable :: skebv_wts (:,:,:) real (kind=kind_phys),allocatable :: sfc_wts (:,:,:) +real (kind=kind_phys),allocatable :: spp_wts (:,:,:,:) integer,allocatable :: blksz(:) integer :: me !< MPI rank designator integer :: root_pe !< MPI rank of root atmosphere processor real(kind=kind_phys) :: dtp !< physics timestep in seconds real(kind=kind_phys) :: fhour !< previous forecast hour real(kind=kind_phys) :: sppt_amp !< amplitude of sppt (to go to cld scheme) -logical :: do_sppt,do_shum,do_skeb,use_zmtnblck -integer :: skeb_npass,n_var_lndp, lndp_type +logical :: do_sppt,do_shum,do_skeb,do_spp,use_zmtnblck +integer :: skeb_npass,n_var_lndp, lndp_type, n_var_spp character(len=65) :: fn_nml !< namelist filename character(len=256),allocatable :: input_nml_file(:) !< character string containing full namelist - namelist /gfs_physics_nml/do_sppt,do_skeb,do_shum,lndp_type,n_var_lndp + namelist /gfs_physics_nml/do_sppt,do_skeb,do_shum,do_spp,lndp_type,n_var_lndp,n_var_spp write_this_tile=.false. ntile_out_str='0' nlunit=23 @@ -80,12 +84,21 @@ program standalone_stochy read(ntile_out_str,'(I1.1)') ntile_out open (unit=nlunit, file='input.nml', status='OLD') n_var_lndp=0 +n_var_spp =0 lndp_type=0 do_sppt=.false. do_shum=.false. do_skeb=.false. +do_spp =.false. read(nlunit,gfs_physics_nml) close(nlunit) + +!allocate spp and lnp arrays +allocate(lndp_var_list(n_var_lndp)) +allocate(lndp_prt_list(n_var_lndp)) +allocate(spp_var_list(n_var_spp)) +allocate(spp_prt_list(n_var_spp)) +allocate(spp_stddev_cutoff(n_var_spp)) ! define stuff pi=3.14159265359 undef=9.99e+20 @@ -178,11 +191,12 @@ program standalone_stochy root_pe=mpp_root_pe() allocate(input_nml_file(1)) input_nml_file='input.nml' -comm=MPI_COMM_WORLD +comm=mpi_comm_world call init_stochastic_physics(nlevs, blksz, dtp, sppt_amp, & input_nml_file, fn_nml, nlunit, xlon, xlat, do_sppt, do_shum, & do_skeb, lndp_type, n_var_lndp, use_zmtnblck, skeb_npass, & lndp_var_list, lndp_prt_list, & + n_var_spp, spp_var_list, spp_prt_list, spp_stddev_cutoff, do_spp, & ak, bk, nthreads, root_pe, comm, ierr) if (ierr .ne. 0) print *, 'ERROR init_stochastic_physics call' ! Draper - need proper error trapping here call get_outfile(fname) @@ -292,6 +306,7 @@ program standalone_stochy if (do_skeb)allocate(skebu_wts(nblks,blksz_1,nlevs)) if (do_skeb)allocate(skebv_wts(nblks,blksz_1,nlevs)) if (lndp_type > 0)allocate(sfc_wts(nblks,blksz_1,n_var_lndp)) +if (do_spp)allocate(spp_wts(nblks,blksz_1,nlevs,n_var_spp)) if (stochini) then istart=11 else @@ -301,7 +316,7 @@ program standalone_stochy do i=istart,21 ts=i/4.0 call run_stochastic_physics(nlevs, i-1, fhour, blksz, & - sppt_wts=sppt_wts, shum_wts=shum_wts, skebu_wts=skebu_wts, skebv_wts=skebv_wts, sfc_wts=sfc_wts, & + sppt_wts=sppt_wts, shum_wts=shum_wts, skebu_wts=skebu_wts, skebv_wts=skebv_wts, sfc_wts=sfc_wts, spp_wts=spp_wts, & nthreads=nthreads) if (me.EQ.0 .and. do_sppt) print*,'SPPT_WTS=',i,sppt_wts(1,1,2) From 7160eb4e27fce29d9b99dfaaef9b40301db8afe9 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 15:21:41 -0400 Subject: [PATCH 24/30] Update Jenkinsfile --- .cicd/Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 056d3e9..bdffc4c 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -17,8 +17,7 @@ pipeline { cd unit_tests echo "Running Unit Tests on ${NODE_NAME}" sbatch run_standalone.sh - cd stochy_out - ls + tail -f out ''' } } From b762de9551b9cf6ec7a24ddf9301563ea896fc93 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 16:07:31 -0400 Subject: [PATCH 25/30] Update Jenkinsfile --- .cicd/Jenkinsfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index bdffc4c..fe8c937 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -17,6 +17,11 @@ pipeline { cd unit_tests echo "Running Unit Tests on ${NODE_NAME}" sbatch run_standalone.sh + pwd + cd .. + pwd + cd unit_tests + ls -l tail -f out ''' } From 539923fe48c878ed2a02e00c72990b17fb8a1c2e Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 16:20:27 -0400 Subject: [PATCH 26/30] Update Jenkinsfile --- .cicd/Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index fe8c937..58e3ae1 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -16,13 +16,13 @@ pipeline { sh ''' cd unit_tests echo "Running Unit Tests on ${NODE_NAME}" - sbatch run_standalone.sh - pwd - cd .. - pwd - cd unit_tests - ls -l - tail -f out + ./compile_standalone.hera_gnu + cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT + srun --label -n 24 standalone_stochy.x + mkdir stochy_out + mv workg* stochy_out + cd stochy_out/ + ls ''' } } From 3f328d1a0e3ef1fe475b5753d682f16ea2351739 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Fri, 17 May 2024 16:21:49 -0400 Subject: [PATCH 27/30] Update Jenkinsfile --- .cicd/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 58e3ae1..e071efe 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -18,7 +18,7 @@ pipeline { echo "Running Unit Tests on ${NODE_NAME}" ./compile_standalone.hera_gnu cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT - srun --label -n 24 standalone_stochy.x + srun --label -A epic -n 24 standalone_stochy.x mkdir stochy_out mv workg* stochy_out cd stochy_out/ From 3d4ee1c3c08731db1577ccd6c4ac436e56b7a763 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:21:40 -0400 Subject: [PATCH 28/30] Update input.nml enable SPP --- unit_tests/input.nml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/unit_tests/input.nml b/unit_tests/input.nml index 723a888..88921b9 100644 --- a/unit_tests/input.nml +++ b/unit_tests/input.nml @@ -56,6 +56,15 @@ / &nam_sppperts + DO_SPP: true + SPP_VAR_LIST: "pbl", "sfc", "mp", "rad", "gwd", "cu_deep" + SPP_MAG_LIST: 0.2, 0.2, 0.75, 0.2, 0.2 + SPP_LSCALE: 150000.0, 150000.0, 150000.0, 150000.0, 150000.0 + SPP_TSCALE: 21600.0, 21600.0, 21600.0, 21600.0, 21600.0 + SPP_SIGTOP1: 0.1, 0.1, 0.1, 0.1, 0.1 + SPP_SIGTOP2: 0.025, 0.025, 0.025, 0.025, 0.025 + SPP_STDDEV_CUTOFF: 1.5, 1.5, 2.5, 1.5, 1.5 + ISEED_SPP: 4, 5, 6, 7, 8 / &nam_stochy From 38498443f9328fde733c51ce9d4fa3547ececa3c Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Mon, 5 Aug 2024 17:10:30 -0400 Subject: [PATCH 29/30] Update input.nml --- unit_tests/input.nml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unit_tests/input.nml b/unit_tests/input.nml index 88921b9..a641fc7 100644 --- a/unit_tests/input.nml +++ b/unit_tests/input.nml @@ -48,15 +48,16 @@ / &gfs_physics_nml + do_spp = .true. do_sppt = .true. do_shum = .true. do_skeb = .true. lndp_type = 2 n_var_lndp = 2 + n_var_spp = 6 / &nam_sppperts - DO_SPP: true SPP_VAR_LIST: "pbl", "sfc", "mp", "rad", "gwd", "cu_deep" SPP_MAG_LIST: 0.2, 0.2, 0.75, 0.2, 0.2 SPP_LSCALE: 150000.0, 150000.0, 150000.0, 150000.0, 150000.0 From 3e6b45502dfe8855d77a169581dd5199a1f4a4c9 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 6 Aug 2024 08:49:31 -0400 Subject: [PATCH 30/30] Update input.nml update n_var_spp --- unit_tests/input.nml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/input.nml b/unit_tests/input.nml index a641fc7..df5fdcd 100644 --- a/unit_tests/input.nml +++ b/unit_tests/input.nml @@ -54,7 +54,7 @@ do_skeb = .true. lndp_type = 2 n_var_lndp = 2 - n_var_spp = 6 + n_var_spp = 5 / &nam_sppperts