Merge branch 'develop' into develop

Author: enrico-usai

Berksfile

@@ -7,15 +7,15 @@ source chef_repo: "."
 
 metadata
 
-cookbook "apt", path: "./third-party/apt-7.4.0"
+cookbook "apt", path: "./third-party/apt-7.4.2"
 cookbook "iptables", path: "./third-party/iptables-8.0.0"
-cookbook "line", path: "./third-party/line-4.0.1"
+cookbook "line", path: "./third-party/line-4.5.2"
 cookbook "nfs", path: "./third-party/nfs-2.6.4"
-cookbook "openssh", path: "./third-party/openssh-2.9.1"
-cookbook "pyenv", path: "./third-party/pyenv-3.4.2"
-cookbook "selinux", path: "./third-party/selinux-3.1.1"
-cookbook "yum", path: "./third-party/yum-6.1.1"
-cookbook "yum-epel", path: "./third-party/yum-epel-4.1.2"
+cookbook "openssh", path: "./third-party/openssh-2.10.3"
+cookbook "pyenv", path: "./third-party/pyenv-3.5.1"
+cookbook "selinux", path: "./third-party/selinux-6.0.4"
+cookbook "yum", path: "./third-party/yum-7.4.0"
+cookbook "yum-epel", path: "./third-party/yum-epel-4.5.0"
 cookbook "aws-parallelcluster-install", path: "./cookbooks/aws-parallelcluster-install"
 cookbook "aws-parallelcluster-config", path: "./cookbooks/aws-parallelcluster-config"
 cookbook "aws-parallelcluster-slurm", path: "./cookbooks/aws-parallelcluster-slurm"

CHANGELOG.md

@@ -3,37 +3,47 @@ aws-parallelcluster-cookbook CHANGELOG
 
 This file is used to list changes made in each version of the AWS ParallelCluster cookbook.
 
-x.x.x
+3.2.0
 ------
 
 **ENHANCEMENTS**
 - Add support for multiple Elastic File Systems.
 - Add support for multiple FSx File System.
 - Add support for attaching existing FSx for Ontap and FSx for OpenZFS File Systems.
 - Install NVIDIA GDRCopy 2.3 to enable low-latency GPU memory copy on supported instance types.
-- Slurm: Set `AuthInfo=cred_expire=70` to reduce the time requeued jobs must wait before starting again when nodes are not available.
-- Make `DirectoryService/AdditionalSssdConfigs` be merged into final SSSD configuration rather than be appended.
-- During cluster update set Slurm nodes state accordingly to strategy set through QueueUpdateStrategy parameter.
-- Add new configuration parameter `Scheduling/SlurmSettings/EnableMemoryBasedScheduling` to configure memory-based
-  scheduling in Slurm.
-  - Move `SelectTypeParameters` and `ConstrainRAMSpace` to the `parallelcluster_slurm*.conf` include files.
-  - Add new configuration parameter to override default value of schedulable memory on compute nodes.
+- During cluster update set Slurm nodes state accordingly to strategy set through the configuration parameter `Scheduling/SchedulerSettings/QueueUpdateStrategy`.
+- Add support for memory-based scheduling in Slurm.
+  - Configure `RealMemory` on compute nodes by default as 95% of the EC2 memory.
+  - Move `SelectTypeParameters` to `slurm_parallelcluster.conf` include file.
+  - Move `ConstrainRAMSpace` to `slurm_parallelcluster_cgroup.conf` include file.
+  - Add support for new configuration parameter `Scheduling/SlurmSettings/EnableMemoryBasedScheduling` to configure memory-based scheduling in Slurm.
+  - Add support for new configuration parameter `Scheduling/SlurmQueues/ComputeResources/SchedulableMemory` to override default value of the memory seen by the scheduler on compute nodes.
 - Add support for rebooting compute nodes via Slurm.
 
 **CHANGES**
-- Slurm: Restart `clustermgtd` and `slurmctld` daemons at cluster update time only when `Scheduling` parameters are updated in the cluster configuration.
+- Restart `clustermgtd` and `slurmctld` daemons at cluster update time only when `Scheduling` parameters are updated in the cluster configuration.
 - Update slurmctld and slurmd systemd service files.
 - Upgrade NICE DCV to version 2022.0-12760.
 - Upgrade NVIDIA driver to version 470.129.06.
 - Upgrade NVIDIA Fabric Manager to version 470.129.06.
-- Upgrade EFA installer to version 1.16.0.
+- Upgrade EFA installer to version 1.17.1.
   - EFA driver: ``efa-1.16.0-1``
   - EFA configuration: ``efa-config-1.10-1``
   - EFA profile: ``efa-profile-1.5-1``
-  - Libfabric: ``libfabric-aws-1.15.1.amzn1.0-1``
-  - RDMA core: ``rdma-core-39.0-2``
-  - Open MPI: ``openmpi40-aws-4.1.2-2``
-- Restrict IPv6 access to IMDS to root and cluster admin users only.
+  - Libfabric: ``libfabric-aws-1.16.0~amzn2.0-1``
+  - RDMA core: ``rdma-core-41.0-2``
+  - Open MPI: ``openmpi40-aws-4.1.4-2``
+- Restrict IPv6 access to IMDS to root and cluster admin users only, when configuration parameter `HeadNode/Imds/Secured` is enabled.
+- Set Slurm configuration `AuthInfo=cred_expire=70` to reduce the time requeued jobs must wait before starting again when nodes are not available.
+- Move `SelectTypeParameters` and `ConstrainRAMSpace` to the `parallelcluster_slurm*.conf` include files.
+- Upgrade third-party cookbook dependencies:
+  - apt-7.4.2 (from apt-7.4.0)
+  - line-4.5.2 (from line-4.0.1)
+  - openssh-2.10.3 (from openssh-2.9.1)
+  - pyenv-3.5.1 (from pyenv-3.4.2)
+  - selinux-6.0.4 (from selinux-3.1.1)
+  - yum-7.4.0 (from yum-6.1.1)
+  - yum-epel-4.5.0 (from yum-epel-4.1.2)
 
 3.1.4
 ------

NOTICE.txt

@@ -1,2 +1,2 @@
 aws-parallelcluster-cookbook
-Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.

attributes/default.rb

@@ -214,7 +214,7 @@
   'default' => 'gdrcopy'
 )
 # EFA
-default['cluster']['efa']['installer_version'] = '1.16.0'
+default['cluster']['efa']['installer_version'] = '1.17.1'
 default['cluster']['efa']['installer_url'] = "https://efa-installer.amazonaws.com/aws-efa-installer-#{node['cluster']['efa']['installer_version']}.tar.gz"
 default['cluster']['efa']['unsupported_aarch64_oses'] = %w(centos7)
 

cookbooks/aws-parallelcluster-awsbatch/metadata.rb

@@ -13,12 +13,12 @@
 supports 'centos', '>= 7.0'
 supports 'ubuntu', '>= 18.04'
 
-depends 'apt', '~> 7.4.0'
+depends 'apt', '~> 7.4.2'
 depends 'iptables', '~> 8.0.0'
-depends 'line', '~> 4.0.1'
+depends 'line', '~> 4.5.2'
 depends 'nfs', '~> 2.6.4'
-depends 'openssh', '~> 2.9.1'
-depends 'pyenv', '~> 3.4.2'
-depends 'selinux', '~> 3.1.1'
-depends 'yum', '~> 6.1.1'
-depends 'yum-epel', '~> 4.1.2'
+depends 'openssh', '~> 2.10.3'
+depends 'pyenv', '~> 3.5.1'
+depends 'selinux', '~> 6.0.4'
+depends 'yum', '~> 7.4.0'
+depends 'yum-epel', '~> 4.5.0'

cookbooks/aws-parallelcluster-config/metadata.rb

@@ -13,12 +13,12 @@
 supports 'centos', '>= 7.0'
 supports 'ubuntu', '>= 18.04'
 
-depends 'apt', '~> 7.4.0'
+depends 'apt', '~> 7.4.2'
 depends 'iptables', '~> 8.0.0'
-depends 'line', '~> 4.0.1'
+depends 'line', '~> 4.5.2'
 depends 'nfs', '~> 2.6.4'
-depends 'openssh', '~> 2.9.1'
-depends 'pyenv', '~> 3.4.2'
-depends 'selinux', '~> 3.1.1'
-depends 'yum', '~> 6.1.1'
-depends 'yum-epel', '~> 4.1.2'
+depends 'openssh', '~> 2.10.3'
+depends 'pyenv', '~> 3.5.1'
+depends 'selinux', '~> 6.0.4'
+depends 'yum', '~> 7.4.0'
+depends 'yum-epel', '~> 4.5.0'

cookbooks/aws-parallelcluster-install/metadata.rb

@@ -13,12 +13,12 @@
 supports 'centos', '>= 7.0'
 supports 'ubuntu', '>= 18.04'
 
-depends 'apt', '~> 7.4.0'
+depends 'apt', '~> 7.4.2'
 depends 'iptables', '~> 8.0.0'
-depends 'line', '~> 4.0.1'
+depends 'line', '~> 4.5.2'
 depends 'nfs', '~> 2.6.4'
-depends 'openssh', '~> 2.9.1'
-depends 'pyenv', '~> 3.4.2'
-depends 'selinux', '~> 3.1.1'
-depends 'yum', '~> 6.1.1'
-depends 'yum-epel', '~> 4.1.2'
+depends 'openssh', '~> 2.10.3'
+depends 'pyenv', '~> 3.5.1'
+depends 'selinux', '~> 6.0.4'
+depends 'yum', '~> 7.4.0'
+depends 'yum-epel', '~> 4.5.0'

cookbooks/aws-parallelcluster-scheduler-plugin/metadata.rb

@@ -13,12 +13,12 @@
 supports 'centos', '>= 7.0'
 supports 'ubuntu', '>= 18.04'
 
-depends 'apt', '~> 7.4.0'
+depends 'apt', '~> 7.4.2'
 depends 'iptables', '~> 8.0.0'
-depends 'line', '~> 4.0.1'
+depends 'line', '~> 4.5.2'
 depends 'nfs', '~> 2.6.4'
-depends 'openssh', '~> 2.9.1'
-depends 'pyenv', '~> 3.4.2'
-depends 'selinux', '~> 3.1.1'
-depends 'yum', '~> 6.1.1'
-depends 'yum-epel', '~> 4.1.2'
+depends 'openssh', '~> 2.10.3'
+depends 'pyenv', '~> 3.5.1'
+depends 'selinux', '~> 6.0.4'
+depends 'yum', '~> 7.4.0'
+depends 'yum-epel', '~> 4.5.0'

cookbooks/aws-parallelcluster-slurm/metadata.rb

@@ -13,12 +13,12 @@
 supports 'centos', '>= 7.0'
 supports 'ubuntu', '>= 18.04'
 
-depends 'apt', '~> 7.4.0'
+depends 'apt', '~> 7.4.2'
 depends 'iptables', '~> 8.0.0'
-depends 'line', '~> 4.0.1'
+depends 'line', '~> 4.5.2'
 depends 'nfs', '~> 2.6.4'
-depends 'openssh', '~> 2.9.1'
-depends 'pyenv', '~> 3.4.2'
-depends 'selinux', '~> 3.1.1'
-depends 'yum', '~> 6.1.1'
-depends 'yum-epel', '~> 4.1.2'
+depends 'openssh', '~> 2.10.3'
+depends 'pyenv', '~> 3.5.1'
+depends 'selinux', '~> 6.0.4'
+depends 'yum', '~> 7.4.0'
+depends 'yum-epel', '~> 4.5.0'

cookbooks/aws-parallelcluster-slurm/templates/default/slurm/compute/slurmd.service.erb

@@ -1,6 +1,6 @@
 [Unit]
 Description=Slurm node daemon
-After=munge.service network.target remote-fs.target
+After=munge.service network-online.target remote-fs.target
 Wants=network-online.target
 ConditionPathExists=<%= node['cluster']['slurm']['install_dir'] %>/etc/slurm.conf
 

cookbooks/aws-parallelcluster-slurm/templates/default/slurm/head_node/slurmctld.service.erb

@@ -1,6 +1,6 @@
 [Unit]
 Description=Slurm controller daemon
-After=network.target munge.service
+After=network-online.target munge.service
 Wants=network-online.target
 ConditionPathExists=<%= node['cluster']['slurm']['install_dir'] %>/etc/slurm.conf
 

metadata.rb

@@ -13,15 +13,15 @@
 supports 'centos', '>= 7.0'
 supports 'ubuntu', '>= 18.04'
 
-depends 'apt', '~> 7.4.0'
+depends 'apt', '~> 7.4.2'
 depends 'iptables', '~> 8.0.0'
-depends 'line', '~> 4.0.1'
+depends 'line', '~> 4.5.2'
 depends 'nfs', '~> 2.6.4'
-depends 'openssh', '~> 2.9.1'
-depends 'pyenv', '~> 3.4.2'
-depends 'selinux', '~> 3.1.1'
-depends 'yum', '~> 6.1.1'
-depends 'yum-epel', '~> 4.1.2'
+depends 'openssh', '~> 2.10.3'
+depends 'pyenv', '~> 3.5.1'
+depends 'selinux', '~> 6.0.4'
+depends 'yum', '~> 7.4.0'
+depends 'yum-epel', '~> 4.5.0'
 depends 'aws-parallelcluster-install', '~> 3.2.0'
 depends 'aws-parallelcluster-config', '~> 3.2.0'
 depends 'aws-parallelcluster-slurm', '~> 3.2.0'

third-party/THIRD-PARTY-LICENSES.txt

@@ -1,15 +1,12 @@
-** apt Cookbook; version 7.4.0 -- https://github.com/chef-cookbooks/apt
-** iptables cookbook; version 8.0.0 --
-https://github.com/chef-cookbooks/iptables
-** line cookbook; version 4.0.1 -- https://github.com/sous-chefs/line
-** NFS cookbook; version 2.6.4 --
-https://github.com/atomic-penguin/cookbook-nfs
-** openssh cookbook; version 2.9.1 -- https://github.com/chef-cookbooks/openssh
-** pyenv cookbook; version 3.4.2 -- https://github.com/sous-chefs/pyenv
-** SELinux cookbook; version 3.1.1 -- https://github.com/chef-cookbooks/selinux
-** yum cookbook; version 6.1.1 -- https://github.com/chef-cookbooks/yum
-** yum-epel cookbook; version 4.1.2 --
-https://github.com/chef-cookbooks/yum-epel
+** apt Cookbook; version 7.4.2 -- https://github.com/sous-chefs/apt
+** iptables cookbook; version 8.0.0 -- https://github.com/sous-chefs/iptables
+** line cookbook; version 4.5.2 -- https://github.com/sous-chefs/line
+** NFS cookbook; version 2.6.4 -- https://github.com/atomic-penguin/cookbook-nfs
+** openssh cookbook; version 2.10.3 -- https://github.com/sous-chefs/openssh
+** pyenv cookbook; version 3.5.1 -- https://github.com/sous-chefs/pyenv
+** SELinux cookbook; version 6.0.4 -- https://github.com/sous-chefs/selinux
+** yum cookbook; version 7.4.0 -- https://github.com/sous-chefs/yum
+** yum-epel cookbook; version 4.5.0 -- https://github.com/sous-chefs/yum-epel
 
 Apache License
 

third-party/apt-7.4.0/.delivery/project.toml

@@ -0,0 +1 @@
+* @sous-chefs/maintainers

third-party/apt-7.4.0/.github/CODEOWNERS

@@ -5,29 +5,21 @@ name: ci
   pull_request:
   push:
     branches:
-      - master
+      - main
 
 jobs:
-  delivery:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@master
-      - name: Run Chef Delivery
-        uses: actionshub/chef-delivery@master
-        env:
-          CHEF_LICENSE: accept-no-persist
+  lint-unit:
+    uses: sous-chefs/.github/.github/workflows/[email protected]
 
-  dokken:
-    needs: [delivery]
+  integration:
+    needs: 'lint-unit'
     runs-on: ubuntu-latest
     strategy:
       matrix:
         os:
-          - 'debian-9'
+          - 'centos-stream-8'
           - 'debian-10'
-          - 'centos-7'
-          - 'ubuntu-1604'
+          - 'debian-11'
           - 'ubuntu-1804'
           - 'ubuntu-2004'
         suite:
@@ -36,31 +28,24 @@ jobs:
           - 'compile-time'
           - 'unattended-upgrades'
         exclude:
-          - os: centos-7
+          - os: centos-stream-8
             suite: cacher
-          - os: centos-7
+          - os: centos-stream-8
             suite: compile-time
-          - os: centos-7
+          - os: centos-stream-8
             suite: unattended-upgrades
       fail-fast: false
 
     steps:
       - name: Check out code
-        uses: actions/checkout@master
+        uses: actions/checkout@v2
       - name: Install Chef
-        uses: actionshub/chef-install@master
+        uses: actionshub/chef-install@main
       - name: Dokken
-        uses: actionshub/kitchen-dokken@master
+        uses: actionshub/test-kitchen@main
         env:
           CHEF_LICENSE: accept-no-persist
           KITCHEN_LOCAL_YAML: kitchen.dokken.yml
         with:
           suite: ${{ matrix.suite }}
           os: ${{ matrix.os }}
-
-  final:
-    needs: [dokken]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@master

third-party/apt-7.4.0/.github/workflows/md-links.yml

@@ -4,14 +4,14 @@ name: md-links
 "on":
   pull_request:
   push:
-    branches: [master]
+    branches: [main]
 
 jobs:
   md-links:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@main
+        uses: actions/checkout@v2
       - name: markdown-link-check
         uses: gaurav-nelson/github-action-markdown-link-check@v1
         with:

third-party/apt-7.4.0/.yamllint

@@ -11,10 +11,6 @@ PreCommit:
     enabled: true
     required_executable: 'cookstyle'
     command: ["cookstyle"]
-  Delivery:
-    enabled: true
-    required_executable: 'delivery'
-    flags: ['local', 'all']
 CommitMsg:
   HardTabs:
     enabled: true