CI: update to container version4.0

- switch to container version 4.0
- re-enable HIP tests disabled with #5246
- add HIP 6.2 and 6.3
- add CUDA 12.6
- add boost 1.86 and 1.87
- adjust filter rules for new CUDA and HIP versions
- adjust filter rules to support ubuntu 22.04 and 24.04

ci: full-compile

.gitlab-ci.yml

@@ -7,7 +7,7 @@ stages:
   - test
 
 variables:
-  CONTAINER_TAG: "3.2"
+  CONTAINER_TAG: "4.0"
   PIC_CI_MAINLINE: "TRUE"
 
 .base_generate-reduced-matrix:
@@ -195,7 +195,7 @@ pypicongpu-compiling-test:
   extends: .base_pypicongpu_compile_test
   variables:
     PIP_BREAK_SYSTEM_PACKAGES: 1
-    CI_CONTAINER_NAME: 'ubuntu20.04'
+    CI_CONTAINER_NAME: 'ubuntu24.04'
     PYTHON_VERSION: '3.11.*'
     CXX_VERSION: 'g++-13'
     CMAKE_VERSION: '3.25'

share/ci/bash.profile

@@ -14,14 +14,6 @@ fi
 export CMAKE_PREFIX_PATH=$PNGWRITER_ROOT:$CMAKE_PREFIX_PATH
 export LD_LIBRARY_PATH=$PNGWRITER_ROOT/lib:$LD_LIBRARY_PATH
 
-export HDF5_ROOT=/opt/hdf5/1.10.7/
-export LD_LIBRARY_PATH=$HDF5_ROOT/lib:$LD_LIBRARY_PATH
-
-export ADIOS2_ROOT=/opt/adios/2.7.1
-export CMAKE_PREFIX_PATH=$ADIOS2_ROOT:$CMAKE_PREFIX_PATH
-export PATH=$ADIOS2_ROOT/bin:$PATH
-export LD_LIBRARY_PATH=$ADIOS2_ROOT/lib:$LD_LIBRARY_PATH
-
 # set environment variable for path to tpls for PyPIConGPU runner
 export PIC_SYSTEM_TEMPLATE_PATH=${PIC_SYSTEM_TEMPLATE_PATH:-"etc/picongpu/bash"}
 

share/ci/install/clang.sh

@@ -7,10 +7,8 @@ set -o pipefail
 
 cd $CI_PROJECT_DIR
 
-# provide sources for clang versions missing in the containers version 3.2
-echo -e "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main\ndeb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main" >> /etc/apt/sources.list.d/llvm.list
-echo -e "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main\ndeb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main" >> /etc/apt/sources.list.d/llvm.list
-echo -e "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main\ndeb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main" >> /etc/apt/sources.list.d/llvm.list
+# provide sources for clang versions missing in the containers version 4.0
+#echo -e "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-17 main\ndeb-src http://apt.llvm.org/noble/ llvm-toolchain-noble-17 main" >> /etc/apt/sources.list.d/llvm.list
 
 apt -y update
 

share/ci/install/openPMD.sh

@@ -3,10 +3,21 @@
 set -e
 set -o pipefail
 
+# ADIOS2 is a dependency of  the pre compiled openPMD
+export ADIOS2_ROOT="$(agc-manager -b [email protected])"
+export CMAKE_PREFIX_PATH=ADIOS2_ROOT:$CMAKE_PREFIX_PATH
+export LD_LIBRARY_PATH=ADIOS2_ROOT/lib:$LD_LIBRARY_PATH
+export PATH=ADIOS2_ROOT/bin:$PATH
+
+# HDF5 is a dependency of  the pre compiled openPMD
+export HDF5_ROOT="$(agc-manager -b [email protected])"
+export CMAKE_PREFIX_PATH=HDF5_ROOT:$CMAKE_PREFIX_PATH
+export LD_LIBRARY_PATH=HDF5_ROOT/lib:$LD_LIBRARY_PATH
+
 if [ -z "$DISABLE_OpenPMD" ] ; then
     cd $CI_PROJECT_DIR
 
-    export openPMD_VERSION=0.15.0
+    export openPMD_VERSION=0.16.1
 
     if ! agc-manager -e openPMD-api@${openPMD_VERSION}; then
         export OPENPMD_ROOT=/opt/openPMD-api/$openPMD_VERSION
@@ -15,7 +26,7 @@ if [ -z "$DISABLE_OpenPMD" ] ; then
         git clone https://github.com/openPMD/openPMD-api.git --depth 1 --branch $openPMD_VERSION
         mkdir buildOpenPMD
         cd buildOpenPMD
-        cmake ../openPMD-api -DCMAKE_CXX_STANDARD=17 -DopenPMD_BUILD_CLI_TOOLS=OFF -DCMAKE_INSTALL_PREFIX=$OPENPMD_ROOT -DopenPMD_BUILD_EXAMPLES=OFF -DopenPMD_BUILD_TESTING=OFF -DopenPMD_USE_PYTHON=OFF -DopenPMD_USE_ADIOS2=ON
+        cmake ../openPMD-api -DCMAKE_CXX_STANDARD=20 -DopenPMD_BUILD_CLI_TOOLS=OFF -DCMAKE_INSTALL_PREFIX=$OPENPMD_ROOT -DopenPMD_BUILD_EXAMPLES=OFF -DopenPMD_BUILD_TESTING=OFF -DopenPMD_USE_PYTHON=OFF -DopenPMD_USE_ADIOS2=ON
         make install
         cd ..
 

share/ci/n_wise_generator.py

@@ -100,10 +100,9 @@ def is_valid_combination(row):
         os_name = row[2][0] if n >= 3 else ""
         os_version = get_version(row[2]) if n >= 3 else 0
 
-        if is_cuda and os_name == "ubuntu":
-            # CI container version 3.1 do not support ubuntu 18.04 anymore
-            if os_version < 20.04:
-                return False
+        # old OS versions base compile mostly does not support C++20
+        if os_name == "ubuntu" and os_version <= 20.04:
+            return False
 
         # CI nvcc image is not shipped with clang++
         # clang_cuda images can currently not be used because
@@ -116,9 +115,11 @@ def is_valid_combination(row):
             if is_hipcc and is_clang:
                 if v_hip < 6.0:
                     return False
-                if 6.0 <= v_hip <= 6.1 and v_compiler == 17:
+                if 6.0 <= v_hip <= 6.1 and v_compiler == 17 and os_version == 22.04:
+                    return True
+                if v_hip == 6.2 and v_compiler == 18 and os_version == 22.04:
                     return True
-                if v_hip == 6.2 and v_compiler == 18:
+                if v_hip == 6.3 and v_compiler == 18 and os_version == 24.04:
                     return True
             return False
 
@@ -127,7 +128,7 @@ def is_valid_combination(row):
             return False
         else:
             # install/clang.sh is currently only providing apt sources up to clang 18
-            if is_clang and v_compiler >= 19:
+            if is_clang and v_compiler >= 20:
                 return False
 
         # CUDA compiler requires backed `cuda`
@@ -140,6 +141,9 @@ def is_valid_combination(row):
 
         # clang cuda compatibility
         if is_clang_cuda:
+            # ubuntu 22.04 is only supported for HIP and is checked before in this filter function
+            if os_name == "ubuntu" and os_version <= 22.04:
+                return False
             if not is_cuda:
                 return False
             if 12.0 <= v_cuda <= 12.3 and v_compiler == 18:
@@ -151,6 +155,9 @@ def is_valid_combination(row):
             # for C++20 support CUDA >= 12 is required
             if v_cuda < 12.0:
                 return False
+            # ubuntu 22.04 is only supported for HIP and is checked before in this filter function
+            if os_name == "ubuntu" and os_version <= 22.04:
+                return False
 
             if is_gnu:
                 # disabled due to a bug in CUDA 12.4 with gcc 13
@@ -164,7 +171,7 @@ def is_valid_combination(row):
                     return False
                 if 12.0 <= v_cuda <= 12.3 and v_compiler <= 12:
                     return True
-                if 12.4 <= v_cuda <= 12.5 and v_compiler <= 13:
+                if 12.4 <= v_cuda <= 12.6 and v_compiler <= 13:
                     return True
 
             if is_clang:
@@ -177,22 +184,39 @@ def is_valid_combination(row):
 
             return False
 
+        # clang as host compiler
+        if is_clang:
+            if os_name == "ubuntu" and os_version == 22.04 and v_compiler <= 12:
+                return True
+            if os_name == "ubuntu" and os_version == 24.04 and v_compiler >= 13:
+                return True
+            return False
+
+        # g++ as host compiler
+        if is_gnu:
+            if os_name == "ubuntu" and os_version == 24.04:
+                return True
+            return False
+
     return True
 
 
 # compiler list
 # tuple with two components (compiler name, version)
 clang_compiers = [
+    # available in apt sources ubuntu 22.04
     ("clang++", 11),
     ("clang++", 12),
+    # available in apt sources ubuntu 24.04
     ("clang++", 13),
     ("clang++", 14),
     ("clang++", 15),
     ("clang++", 16),
     ("clang++", 17),
     ("clang++", 18),
+    ("clang++", 19),
 ]
-gnu_compilers = [("g++", 10), ("g++", 11), ("g++", 13)]
+gnu_compilers = [("g++", 10), ("g++", 11), ("g++", 13), ("g++", 14)]
 compilers = [clang_compiers, gnu_compilers]
 
 # generate clang cuda compiler list
@@ -220,14 +244,17 @@ def is_valid_combination(row):
 # tuple with two components (backend name, version)
 # version is only required for the cuda backend
 backends = [
-    # ("hip", 6.0),
-    # ("hip", 6.1),
+    ("hip", 6.0),
+    ("hip", 6.1),
+    ("hip", 6.2),
+    ("hip", 6.3),
     ("cuda", 12.0),
     ("cuda", 12.1),
     ("cuda", 12.2),
     ("cuda", 12.3),
     ("cuda", 12.4),
     ("cuda", 12.5),
+    ("cuda", 12.6),
     ("omp2b",),
     ("serial",),
 ]
@@ -245,9 +272,11 @@ def is_valid_combination(row):
     "1.83.0",
     "1.84.0",
     "1.85.0",
+    "1.86.0",
+    "1.87.0",
 ]
 
-operating_system = [("ubuntu", 20.04)]
+operating_system = [("ubuntu", 22.04), ("ubuntu", 24.04)]
 
 if args.limit_boost_versions:
     # select each second but keep the order

share/ci/pypicongpu_generator.py

@@ -325,7 +325,7 @@ def print_job_yaml(test_pkg_versions: Dict[str, List[str]]):
             body={
                 "variables": {
                     "PYTHON_VERSION": pyVer + ".*",
-                    "CI_CONTAINER_NAME": "ubuntu20.04",
+                    "CI_CONTAINER_NAME": "ubuntu24.04",
                 },
                 "extends": ".base_pypicongpu_quick_test",
             },