diff --git a/compass/machines/chicoma-cpu.cfg b/compass/machines/chicoma-cpu.cfg new file mode 100644 index 0000000000..600a768e5e --- /dev/null +++ b/compass/machines/chicoma-cpu.cfg @@ -0,0 +1,54 @@ + +# The paths section describes paths that are used within the ocean core test +# cases. +[paths] + +# A shared root directory where MPAS standalone data can be found +database_root = /usr/projects/regionalclimate/COMMON_MPAS/mpas_standalonedata/ + +# the path to the base conda environment where compass environments have +# been created +compass_envs = /usr/projects/climate/SHARED_CLIMATE/compass/chicoma-cpu/base + + +# Options related to deploying a compass conda environment on supported +# machines +[deploy] + +# the compiler set to use for system libraries and MPAS builds +compiler = gnu + +# the system MPI library to use for gnu compiler +mpi_gnu = mpich + +# the base path for spack environments used by compass +spack = /usr/projects/climate/SHARED_CLIMATE/compass/chicoma-cpu/spack + +# whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and +# pnetcdf as E3SM (spack modules are used otherwise) +use_e3sm_hdf5_netcdf = True + + +# The parallel section describes options related to running jobs in parallel +[parallel] + +# account for running diagnostics jobs +account = + +# cores per node on the machine +cores_per_node = 128 + +# threads per core (set to 1 because trying to hyperthread seems to be causing +# hanging on perlmutter) +threads_per_core = 1 + + +# Config options related to creating a job script +[job] + +# The job partition to use +partition = standard + +# The job quality of service (QOS) to use +qos = standard + diff --git a/conda/albany_supported.txt b/conda/albany_supported.txt index d40679c7b7..b77f98d25f 100644 --- a/conda/albany_supported.txt +++ b/conda/albany_supported.txt @@ -1,6 +1,7 @@ # a list of supported machine, compiler and mpi combinations for Albany anvil, gnu, openmpi +chicoma-cpu, gnu, mpich chrysalis, gnu, openmpi compy, gnu, openmpi cori-haswell, gnu, mpt diff --git a/conda/petsc_supported.txt b/conda/petsc_supported.txt index 16d4a7e897..98a64b08bc 100644 --- a/conda/petsc_supported.txt +++ b/conda/petsc_supported.txt @@ -3,6 +3,7 @@ anvil, intel, impi anvil, gnu, openmpi +chicoma-cpu, gnu, mpich chrysalis, intel, openmpi chrysalis, gnu, openmpi compy, intel, impi diff --git a/conda/unsupported.txt b/conda/unsupported.txt index 0f4c6ead20..4e72cffc0a 100644 --- a/conda/unsupported.txt +++ b/conda/unsupported.txt @@ -2,6 +2,9 @@ # no spack available anvil, gnu, impi +chicoma-cpu, nvidia, mpich +chicoma-cpu, aocc, mpich +chicoma-cpu, amdclang, mpich chrysalis, gnu, impi chrysalis, oneapi-ifx, openmpi chrysalis, oneapi-ifx, impi diff --git a/docs/developers_guide/machines/chicoma.rst b/docs/developers_guide/machines/chicoma.rst new file mode 100644 index 0000000000..ed4e01dae0 --- /dev/null +++ b/docs/developers_guide/machines/chicoma.rst @@ -0,0 +1,18 @@ +Chicoma +======= + +chicoma-cpu, gnu +---------------- + +If you've set things up for this compiler, you should be able to source a load +script similar to: + +.. code-block:: bash + + source load_dev_compass_1.2.0-alpha.4_chicoma-cpu_gnu_mpich.sh + +Then, you can build the MPAS model with + +.. code-block:: bash + + make [DEBUG=true] gnu-cray diff --git a/docs/developers_guide/machines/index.rst b/docs/developers_guide/machines/index.rst index 417a5565c5..1f050d36f2 100644 --- a/docs/developers_guide/machines/index.rst +++ b/docs/developers_guide/machines/index.rst @@ -51,6 +51,8 @@ the MPAS model. | | +-----------+-------------------+ | | | mvapich | gfortran | +--------------+------------+-----------+-------------------+ +| chicoma-cpu | gnu | mpich | gnu-cray | ++--------------+------------+-----------+-------------------+ | chrysalis | intel | openmpi | ifort | | | +-----------+-------------------+ | | | impi | intel-mpi | @@ -74,6 +76,7 @@ Below are specifics for each supported machine :titlesonly: anvil + chicoma chrysalis compy cori diff --git a/docs/users_guide/machines/chicoma.rst b/docs/users_guide/machines/chicoma.rst new file mode 100644 index 0000000000..b2fc6ff3ea --- /dev/null +++ b/docs/users_guide/machines/chicoma.rst @@ -0,0 +1,192 @@ +.. _machine_chicoma: + +Chicoma +======= + +`LANL IC overview and search `_ + +`DST Calendar `_ (within LANL network) + +Information about Slurm: + +* `Introduction to Slurm at LANL `_ + +* `Basic Slurm Guide for LANL HPC Users `_ + +* `Slurm Command Summary `_ + +* `Slurm: Running Jobs on HPC Platforms `_ + +* `example of batch scripts `_ + +Machine specifications: `chicoma `_ +`turquoise network `_ + +login: ``ssh -t @wtrw.lanl.gov ssh ch-fe`` + +File locations: + +* small home directory, for start-up scripts only: ``/users/`` + +* home directory, backed up: ``/usr/projects/climate/`` + +* scratch space, not backed up: ``/lustre/scratch4/turquoise/`` or + ``scratch5`` + +Check compute time: + +* ``sacctmgr list assoc user= format=Cluster,Account%18,Partition,QOS%45`` + +* Which is my default account? ``sacctmgr list user `` + +* ``sshare -a | head -2; sshare -a | grep $ACCOUNT | head -1`` + +* ``sreport -t Hours cluster AccountUtilizationByUser start=2019-12-02 | grep $ACCOUNT`` + +* check job priority: ``sshare -a | head -2; sshare -a | grep $ACCOUNT`` + +* `LANL Cluster Usage Overview `_ (within LANL yellow) + +Check disk usage: + +* your home space: ``chkhome`` + +* total disk usage in Petabytes: ``df -BP |head -n 1; df -BP|grep climate; df -BP |grep scratch`` + +Archiving + +* `turquoise HPSS archive `_ + +* archive front end: ``ssh -t @wtrw.lanl.gov ssh ar-tn`` + +* storage available at: ``cd /archive/`` + +* you can just copy files directly into here for a particular project. + +LANL uses slurm. To obtain an interactive node: + +.. code-block:: bash + salloc -N 1 -t 2:0:0 --qos=interactive +Use ``--account=ACCOUNT_NAME`` to change to a particular account. + +Chicoma-CPU +----------- + +**There has not yet been a release with Chicoma-CPU, so the following applies +to the release of compass v1.2.0, when it happens.** + +Chicoma's CPU and GPU nodes have different configuration options and compilers. +We only support Chicoma-CPU at this time. + +config options +~~~~~~~~~~~~~~ + +Here are the default config options added when you choose ``-m chicoma-cpu`` +when setting up test cases or a test suite: + +.. code-block:: cfg + + # The paths section describes paths that are used within the ocean core test + # cases. + [paths] + + # A shared root directory where MPAS standalone data can be found + database_root = /usr/projects/regionalclimate/COMMON_MPAS/mpas_standalonedata/ + + # the path to the base conda environment where compass environments have + # been created + compass_envs = /usr/projects/climate/SHARED_CLIMATE/compass/chicoma-cpu/base + + + # Options related to deploying a compass conda environment on supported + # machines + [deploy] + + # the compiler set to use for system libraries and MPAS builds + compiler = gnu + + # the system MPI library to use for gnu compiler + mpi_gnu = mpich + + # the base path for spack environments used by compass + spack = /usr/projects/climate/SHARED_CLIMATE/compass/chicoma-cpu/spack + + # whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and + # pnetcdf as E3SM (spack modules are used otherwise) + use_e3sm_hdf5_netcdf = True + + + # The parallel section describes options related to running jobs in parallel + [parallel] + + # account for running diagnostics jobs + account = + + # cores per node on the machine + cores_per_node = 128 + + # threads per core (set to 1 because trying to hyperthread seems to be causing + # hanging on chicoma) + threads_per_core = 1 + + + # Config options related to creating a job script + [job] + + # The job partition to use + partition = standard + + # The job quality of service (QOS) to use + qos = standard + + +Additionally, some relevant config options come from the +`mache `_ package: + +.. code-block:: cfg + + # The parallel section describes options related to running jobs in parallel + [parallel] + + # parallel system of execution: slurm, cobalt or single_node + system = slurm + + # whether to use mpirun or srun to run a task + parallel_executable = srun + + # cores per node on the machine + cores_per_node = 256 + + # available partition(s) (default is the first) + partitions = standard, gpu + + # quality of service (default is the first) + qos = standard, debug + + + # Config options related to spack environments + [spack] + + # whether to load modules from the spack yaml file before loading the spack + # environment + modules_before = False + + # whether to load modules from the spack yaml file after loading the spack + # environment + modules_after = False + +Gnu on Chicoma-CPU +~~~~~~~~~~~~~~~~~~ + +To load the compass environment and modules, and set appropriate environment +variables: + +.. code-block:: bash + + source /usr/projects/climate/SHARED_CLIMATE/compass/chicoma-cpu/load_latest_compass_gnu_mpich.sh + +To build the MPAS model with + +.. code-block:: bash + + make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-cray diff --git a/docs/users_guide/machines/index.rst b/docs/users_guide/machines/index.rst index 84035514f1..0e76585c49 100644 --- a/docs/users_guide/machines/index.rst +++ b/docs/users_guide/machines/index.rst @@ -133,6 +133,7 @@ to use the default MPI variant for each compiler on each machine. :titlesonly: anvil + chicoma chrysalis compy cori diff --git a/docs/users_guide/quick_start.rst b/docs/users_guide/quick_start.rst index e0d6a562c5..c103cb1ca8 100644 --- a/docs/users_guide/quick_start.rst +++ b/docs/users_guide/quick_start.rst @@ -26,6 +26,12 @@ Here are the commands to load the the environment for the latest source /lcrc/soft/climate/compass/anvil/load_latest_compass.sh +* Chicoma-CPU (coming soon): + +.. code-block:: bash + + source /usr/projects/climate/SHARED_CLIMATE/compass/chicoma-cpu/load_latest_compass.sh + * Chrysalis: .. code-block:: bash diff --git a/utils/matrix/setup_matrix.py b/utils/matrix/setup_matrix.py index 1c79632364..735d584175 100755 --- a/utils/matrix/setup_matrix.py +++ b/utils/matrix/setup_matrix.py @@ -28,6 +28,8 @@ ('intel', 'mvapich'): 'ifort', ('gnu', 'openmpi'): 'gfortran', ('gnu', 'mvapich'): 'gfortran'}, + 'chicoma-cpu': { + ('gnu', 'mpich'): 'gnu-cray'}, 'chrysalis': { ('intel', 'impi'): 'intel-mpi', ('intel', 'openmpi'): 'ifort',