Port land-DA workflow to Hercules (#133)

* add module files for hercules

* add ctest scripts

* add yaml file

* add we2e to pr template

* fix typo in ctest

* rm dummy line

* adjust time limit

.github/pull_request_template.md

@@ -27,6 +27,8 @@ EXAMPLE: Closes ufs-community/land-DA/issues/<issue_number>
     - [ ] Hercules
 - CI
   - [ ] Completed
+- WE2E
+  - [ ] Completed
 - PW-Clouds
   - [ ] AWS
   - [ ] AZURE

modulefiles/build_hercules_intel.lua

@@ -0,0 +1,46 @@
+help([[
+loads modules necessary for building the land-DA workflow on Hercules using Intel
+]])
+
+whatis([===[Loads modules necessary for building the land-DA workflow on Hercules]===])
+
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("cmake", cmake_ver))
+load(pathJoin("ecbuild", ecbuild_ver))
+
+load(pathJoin("jasper", jasper_ver))
+load(pathJoin("zlib", zlib_ver))
+load(pathJoin("libpng", libpng_ver))
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("parallelio", parallelio_ver))
+load(pathJoin("esmf", esmf_ver))
+load(pathJoin("fms",fms_ver))
+load(pathJoin("bacio", bacio_ver))
+load(pathJoin("crtm", crtm_ver))
+load(pathJoin("g2", g2_ver))
+load(pathJoin("g2tmpl", g2tmpl_ver))
+load(pathJoin("ip", ip_ver))
+load(pathJoin("sp", sp_ver))
+load(pathJoin("w3emc", w3emc_ver))
+load(pathJoin("gftl-shared", gftl_shared_ver))
+load(pathJoin("mapl", mapl_ver))
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+
+setenv("CFLAGS","-diag-disable=10441")
+setenv("FFLAGS","-diag-disable=10441")
+
+setenv("CC", "mpiicc")
+setenv("CXX", "mpiicpc")
+setenv("FC", "mpiifort")
+setenv("CMAKE_Platform", "hercules.intel")
+
+setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA_Dev")
+setenv("JEDI_INSTALL", "/work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules")

modulefiles/tasks/hercules/task.analysis.lua

@@ -0,0 +1,10 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hercules/task.forecast.lua

@@ -0,0 +1,28 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("jasper", jasper_ver))
+load(pathJoin("zlib", zlib_ver))
+load(pathJoin("libpng", libpng_ver))
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("parallelio", parallelio_ver))
+load(pathJoin("esmf", esmf_ver))
+load(pathJoin("fms",fms_ver))
+load(pathJoin("bacio", bacio_ver))
+load(pathJoin("crtm", crtm_ver))
+load(pathJoin("g2", g2_ver))
+load(pathJoin("g2tmpl", g2tmpl_ver))
+load(pathJoin("ip", ip_ver))
+load(pathJoin("sp", sp_ver))
+load(pathJoin("w3emc", w3emc_ver))
+load(pathJoin("gftl-shared", gftl_shared_ver))
+load(pathJoin("mapl", mapl_ver))
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hercules/task.plot_stats.lua

@@ -0,0 +1,15 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-cartopy", py_cartopy_ver))
+load(pathJoin("py-matplotlib", py_matplotlib_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+load(pathJoin("py-pyyaml", py_pyyaml_ver))
+load(pathJoin("py-scipy", py_scipy_ver))
+load(pathJoin("py-xarray", py_xarray_ver))
+

modulefiles/tasks/hercules/task.post_anal.lua

@@ -0,0 +1,13 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+

modulefiles/tasks/hercules/task.pre_anal.lua

@@ -0,0 +1,10 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("prod_util", prod_util_ver))

modulefiles/tasks/hercules/task.prep_obs.lua

@@ -0,0 +1,7 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))

modulefiles/wflow_hercules.lua

@@ -0,0 +1,18 @@
+help([[
+This module loads python environement for running the land-DA workflow on
+the MSU machine Hercules
+]])
+
+whatis([===[Loads libraries needed for running the land-DA workflow on Hercules ]===])
+
+load("contrib")
+load("rocoto")
+
+unload("python")
+load("conda")
+
+if mode() == "load" then
+   LmodMsgRaw([===[Please do the following to activate conda:
+       > conda activate land_da
+]===])
+end

parm/land_analysis_hera.yaml

@@ -7,8 +7,7 @@ workflow:
   cycledef:
     - attrs:
         group: cycled
-      spec: 200001030000 200001040000 24:00:00   # for gswp3
-#      spec: 201912210000 201912220000 24:00:00   # for era5
+      spec: 200001030000 200001040000 24:00:00
   entities:
     MACHINE: "hera"
     SCHED: "slurm"