From 74065a0c1982ca1ed943367e20d9455d17938319 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Jul 2024 17:47:30 +0000 Subject: [PATCH 1/5] Use replace_tmpl instead of namespaces in archiving --- parm/archive/master_enkf.yaml.j2 | 40 +++++++++++--------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 70f8a2ad89..3ebd52dbad 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -52,35 +52,21 @@ datasets: {% for mem in range(first_group_mem, last_group_mem + 1) %} # Declare a dict of search and replace terms to run on each template - {% set tmpl_dict = {'ROTDIR':ROTDIR, - 'RUN':RUN, - 'YMD':cycle_YMD, - 'HH':cycle_HH, - 'MEMDIR':"mem" + '%03d' % mem} %} - - # Replace template variables with tmpl_dict, one key at a time - # This must be done in a namespace to overcome jinja scoping - # Variables set inside of a for loop are lost at the end of the loop - # unless they are part of a namespace - {% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL, - COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL, - COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %} - - {% for key in tmpl_dict.keys() %} - {% set search_term = '${' + key + '}' %} - {% set replace_term = tmpl_dict[key] %} - {% set com_ns.COMIN_ATMOS_ANALYSIS_MEM = - com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %} - {% set com_ns.COMIN_ATMOS_HISTORY_MEM = - com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %} - {% set com_ns.COMIN_ATMOS_RESTART_MEM = - com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %} - {% endfor %} + {% set mem_char = 'mem%03d' | format(mem) %} + {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':cycle_YMD, + '${HH}':cycle_HH, + '${MEMDIR}': mem_char }) %} + + {% set COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} + {% set COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) %} + {% set COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %} # Append the member COM directories - {% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%} - {% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%} - {% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%} + {% do COMIN_ATMOS_ANALYSIS_MEM_list.append(COMIN_ATMOS_ANALYSIS_MEM)%} + {% do COMIN_ATMOS_HISTORY_MEM_list.append(COMIN_ATMOS_HISTORY_MEM)%} + {% do COMIN_ATMOS_RESTART_MEM_list.append(COMIN_ATMOS_RESTART_MEM)%} {% endfor %} From 1335dfffed9fd5a2d624c111a97ab8bca3fd235c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 19 Jul 2024 15:08:59 +0000 Subject: [PATCH 2/5] Do not specify a service account --- workflow/hosts/awspw.yaml | 1 - workflow/hosts/container.yaml | 1 - workflow/hosts/gaea.yaml | 1 - workflow/hosts/hera.yaml | 1 - workflow/hosts/hercules.yaml | 1 - workflow/hosts/jet.yaml | 1 - workflow/hosts/orion.yaml | 1 - workflow/hosts/s4.yaml | 1 - workflow/hosts/wcoss2.yaml | 1 - workflow/rocoto/tasks.py | 2 +- 10 files changed, 1 insertion(+), 10 deletions(-) diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index b7021a6e3f..046dafcfa7 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -7,7 +7,6 @@ STMP: '/lustre/${USER}/stmp2/' PTMP: '/lustre/${USER}/stmp4/' NOSCRUB: ${HOMEDIR} ACCOUNT: hwufscpldcld -ACCOUNT_SERVICE: hwufscpldcld SCHEDULER: slurm QUEUE: batch QUEUE_SERVICE: batch diff --git a/workflow/hosts/container.yaml b/workflow/hosts/container.yaml index 907f69754e..d7924724ae 100644 --- a/workflow/hosts/container.yaml +++ b/workflow/hosts/container.yaml @@ -8,7 +8,6 @@ PTMP: '/home/${USER}' NOSCRUB: $HOMEDIR SCHEDULER: none ACCOUNT: '' -ACCOUNT_SERVICE: '' QUEUE: '' QUEUE_SERVICE: '' PARTITION_BATCH: '' diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index ff9877e77b..619a86f2e5 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -9,7 +9,6 @@ STMP: '/gpfs/f5/ufs-ard/scratch/${USER}' PTMP: '/gpfs/f5/ufs-ard/scratch/${USER}' NOSCRUB: $HOMEDIR ACCOUNT: ufs-ard -ACCOUNT_SERVICE: ufs-ard SCHEDULER: slurm QUEUE: normal QUEUE_SERVICE: normal diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 76a7158f43..731e583961 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -8,7 +8,6 @@ STMP: '/scratch1/NCEPDEV/stmp2/${USER}' PTMP: '/scratch1/NCEPDEV/stmp4/${USER}' NOSCRUB: $HOMEDIR ACCOUNT: fv3-cpu -ACCOUNT_SERVICE: fv3-cpu SCHEDULER: slurm QUEUE: batch QUEUE_SERVICE: batch diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index 975558160f..b513bfd57a 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -9,7 +9,6 @@ PTMP: '/work/noaa/stmp/${USER}/HERCULES' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu -ACCOUNT_SERVICE: fv3-cpu QUEUE: batch QUEUE_SERVICE: batch PARTITION_BATCH: hercules diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index b526e073c3..ae7267d687 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -8,7 +8,6 @@ STMP: '/lfs4/HFIP/hfv3gfs/${USER}/stmp' PTMP: '/lfs4/HFIP/hfv3gfs/${USER}/ptmp' NOSCRUB: $HOMEDIR ACCOUNT: hfv3gfs -ACCOUNT_SERVICE: hfv3gfs SCHEDULER: slurm QUEUE: batch QUEUE_SERVICE: batch diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index fe36c8e7ce..f0f807aacf 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -9,7 +9,6 @@ PTMP: '/work/noaa/stmp/${USER}/ORION' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu -ACCOUNT_SERVICE: fv3-cpu QUEUE: batch QUEUE_SERVICE: batch PARTITION_BATCH: orion diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index 37479fa13c..aea807da63 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -8,7 +8,6 @@ STMP: '/scratch/users/${USER}' PTMP: '/scratch/users/${USER}' NOSCRUB: ${HOMEDIR} ACCOUNT: star -ACCOUNT_SERVICE: star SCHEDULER: slurm QUEUE: s4 QUEUE_SERVICE: serial diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index e3650e4710..7ae2be1424 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -8,7 +8,6 @@ STMP: '/lfs/h2/emc/stmp/${USER}' PTMP: '/lfs/h2/emc/ptmp/${USER}' NOSCRUB: $HOMEDIR ACCOUNT: 'GFS-DEV' -ACCOUNT_SERVICE: 'GFS-DEV' SCHEDULER: pbspro QUEUE: 'dev' QUEUE_SERVICE: 'dev_transfer' diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 097d1adef5..404203f02d 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -176,7 +176,7 @@ def get_resource(self, task_name): task_config = self._configs[task_name] - account = task_config['ACCOUNT_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config['ACCOUNT'] + account = task_config['ACCOUNT'] if f'wtime_{task_name}_{self.cdump}' in task_config: walltime = task_config[f'wtime_{task_name}_{self.cdump}'] From cd62bea63e067265072e683829468fb6e0693c7c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 19 Jul 2024 15:28:39 +0000 Subject: [PATCH 3/5] Perform shallow submodule update to save disk space --- .github/workflows/ci_unit_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_unit_tests.yaml b/.github/workflows/ci_unit_tests.yaml index e22f63bf56..6dbc7ee52c 100644 --- a/.github/workflows/ci_unit_tests.yaml +++ b/.github/workflows/ci_unit_tests.yaml @@ -47,7 +47,7 @@ jobs: run: | sudo mkdir -p /scratch1/NCEPDEV cd $GITHUB_WORKSPACE/sorc - git submodule update --init --recursive + git submodule update --init ./link_workflow.sh cd $GITHUB_WORKSPACE/ci/scripts/tests ln -s ../wxflow From d7d3c62043a05e9c788d924e31c0e4c8cb7a462b Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 19 Jul 2024 18:39:56 +0000 Subject: [PATCH 4/5] Skip LFS smudge on checkout --- ci/scripts/clone-build_ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index 3cef7fc230..6ffcf5d804 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -44,6 +44,7 @@ if [[ -d global-workflow ]]; then rm -Rf global-workflow fi +export GIT_LFS_SKIP_SMUDGE=1 git clone "${REPO_URL}" cd global-workflow || exit 1 From ba8e1e8ffde9755a4ffcf473543ec5e4c340731f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 22 Jul 2024 14:06:06 +0000 Subject: [PATCH 5/5] Reallow git-lfs on CI builds --- ci/scripts/clone-build_ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index 6ffcf5d804..3cef7fc230 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -44,7 +44,6 @@ if [[ -d global-workflow ]]; then rm -Rf global-workflow fi -export GIT_LFS_SKIP_SMUDGE=1 git clone "${REPO_URL}" cd global-workflow || exit 1