hpc-slurm-ramble-gromacs.yaml

# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---

blueprint_name: hpc-slurm-ramble-gromacs

vars:
  project_id:  ## Set GCP Project ID Here ##
  deployment_name: hpc-slurm-ramble-gromacs
  region: us-central1
  zone: us-central1-c
  system_user_name: spack-ramble

# Documentation for each of the modules used below can be found at
# https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md

deployment_groups:
- group: primary
  modules:
  # Source is an embedded module, denoted by "modules/*" without ./, ../, /
  # as a prefix. To refer to a local module, prefix with ./, ../ or /
  # Example - ./modules/network/vpc
  - id: network1
    source: modules/network/vpc

  ## Install Scripts
  - id: spack-install
    source: community/modules/scripts/spack-setup
    settings:
      install_dir: /opt/apps/spack

  - id: spack-execute
    source: community/modules/scripts/spack-execute
    use: [spack-install]
    settings:
      log_file: /var/log/spack.log
      commands: |
        # Un-comment and update mirror_url to install from spack cache
        # if ! spack mirror list | grep -q gcs_cache; then
        #   spack mirror add --scope site gcs_cache gs://optionally_set_spack_cache_bucket
        # fi
        # spack buildcache keys --install --trust

        spack config --scope defaults add config:build_stage:/opt/apps/spack/spack-stage
        spack config --scope defaults add concretizer:targets:host_compatible:false

  - id: ramble-setup
    source: community/modules/scripts/ramble-setup
    settings:
      install_dir: /opt/apps/ramble

  - id: ramble-execute
    source: community/modules/scripts/ramble-execute
    use: [spack-install, ramble-setup]
    settings:
      commands: |
        ramble workspace create gromacs -c /opt/apps/gromacs_scaling.yaml -t /opt/apps/execute_experiment.tpl
        ramble workspace activate gromacs
        ramble workspace info -v

  - id: login-startup
    source: modules/scripts/startup-script
    settings:
      runners:
      - $(spack-execute.spack_runner)
      - type: data
        destination: /opt/apps/gromacs_scaling.yaml
        content: |
          ramble:
            variables:
              mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}'
              batch_submit: 'sbatch {execute_experiment}'
              processes_per_node: 30
            applications:
              gromacs:
                workloads:
                  lignocellulose:
                    experiments:
                      scaling_{n_nodes}:
                        variables:
                          n_nodes: [1, 2, 4, 8, 16]
            spack:
              concretized: false
              packages: {}
              environments: {}
      - type: data
        destination: /opt/apps/execute_experiment.tpl
        content: |
          #!/bin/bash
          #SBATCH -N {n_nodes}
          #SBATCH --ntasks-per-node {processes_per_node}
          cd "{experiment_run_dir}"
          {command}
      - $(ramble-execute.ramble_runner)

  - id: access-startup
    source: modules/scripts/startup-script
    settings:
      runners:
      - $(ramble-setup.ramble_runner)

  - id: compute_node_group
    source: community/modules/compute/schedmd-slurm-gcp-v5-node-group
    settings:
      node_count_dynamic_max: 20
      bandwidth_tier: gvnic_enabled

  - id: compute_partition
    source: community/modules/compute/schedmd-slurm-gcp-v5-partition
    use:
    - network1
    - compute_node_group
    settings:
      partition_name: compute

  - id: slurm_controller
    source: community/modules/scheduler/schedmd-slurm-gcp-v5-controller
    use:
    - network1
    - compute_partition
    - access-startup
    settings:
      disable_controller_public_ips: false

  - id: slurm_login
    source: community/modules/scheduler/schedmd-slurm-gcp-v5-login
    use:
    - network1
    - slurm_controller
    - login-startup
    settings:
      machine_type: n2-standard-4
      disable_login_public_ips: false