Add compatibility with HIP (#289)

Co-authored-by: Hanno Spreeuw <[email protected]> Co-authored-by: Leon Oostrum <[email protected]> Co-authored-by: John Romein <[email protected]>
nlesc-recruit · Jul 19, 2024 · c55a698 · c55a698
1 parent 499cc57
commit c55a698
Show file tree

Hide file tree

Showing 19 changed files with 1,134 additions and 85 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -7,10 +7,11 @@ on:
 jobs:
   linting:
     name: run
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
-        gcc-version: "12" 
         cuda-version: "12.2.0"
+        hip-version: "6.1.3"
+        amdgpu-version: "6.1.60103-1_all"
     steps:
 
       - name: Check out a copy of the repository
@@ -33,17 +34,29 @@ jobs:
           non-cuda-sub-packages: '["libcufft-dev"]'
           linux-local-args: '["--toolkit"]'
 
+      - name: Install ROCm and HIP
+        shell: bash -el {0}
+        run: |
+          wget https://repo.radeon.com/amdgpu-install/${{ env.hip-version }}/ubuntu/jammy/amdgpu-install_${{ env.amdgpu-version }}.deb
+          sudo apt -y install ./amdgpu-install_${{ env.amdgpu-version }}.deb
+          sudo apt update
+          sudo apt -y install rocm-hip-runtime-dev hipfft-dev
+
       - name: Show tool versions
         run: |
            echo "nvcc: $(nvcc --version)"
-           echo "g++: $(g++-${{ env.gcc-version}} --version)"
+           echo "hipcc: $(/opt/rocm/bin/hipcc --version)"
+           echo "g++: $(g++ --version)"
+           echo "cmake: $(cmake --version)"
 
-      - name: Run CMake
+      - name: Build the code in CUDA mode
         shell: bash -el {0}
         run: |
-          cmake -DCMAKE_CXX_COMPILER=g++-${{ env.gcc-version }} -DCUDAWRAPPERS_BUILD_TESTING=True -S . -B build
+          cmake -DCUDAWRAPPERS_BUILD_TESTING=True -S . -B build
+          make -C build -j
 
-      - name: Build the code
+      - name: Build the code in HIP mode
         shell: bash -el {0}
         run: |
-          cd build && make -j
+          cmake -DCUDAWRAPPERS_BUILD_TESTING=True -DCUDAWRAPPERS_BACKEND=HIP -DAMDGPU_TARGETS=gfx1101 -S . -B build
+          make -C build -j
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,6 +3,7 @@ repos:
     rev: v18.1.8
     hooks:
       - id: clang-format
+        exclude: macros.hpp
   - repo: https://github.com/pocc/pre-commit-hooks
     rev: v1.3.5
     hooks:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,8 +5,14 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+### Added
+
+- Added compatibility with HIP
+- Added `cu::Device::getArch()`
+
 ### Changed
 
+- `cu::Context::{getCurrent, popCurrent, getDevice}` are no longer static
 - `inline_local_includes` is now more robust: it properly handles commented
   includes and respects the location of an include in the original source file
 

diff --git a/CITATION.cff b/CITATION.cff
@@ -37,6 +37,11 @@ authors:
     email: [email protected]
     affiliation: Netherlands eScience Center
     orcid: 'https://orcid.org/0000-0001-8724-8372'
+  - given-names: Hanno
+    family-names: Spreeuw
+    email: [email protected]
+    affiliation: Netherlands eScience Center
+    orcid: 'https://orcid.org/0000-0002-5057-0322'
   - given-names: Laura
     family-names: Ootes
     email: [email protected]
@@ -76,6 +81,7 @@ repository-code: 'https://github.com/nlesc-recruit/cudawrappers'
 abstract: C++ Wrappers for the CUDA Driver API.
 keywords:
   - CUDA
+  - HIP
   - C++
   - GPU
   - accelerators

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -27,6 +27,31 @@ option(CUDAWRAPPERS_BUILD_TESTING "Build cudawrappers tests"
        ${CUDAWRAPPERS_TESTING_DEFAULT}
 )
 
+if(NOT DEFINED CUDAWRAPPERS_BACKEND)
+  set(CUDAWRAPPERS_BACKEND "CUDA")
+endif()
+set(CUDAWRAPPERS_BACKEND
+    ${CUDAWRAPPERS_BACKEND}
+    CACHE STRING "Backend API to use in cudawrappers"
+)
+set_property(CACHE CUDAWRAPPERS_BACKEND PROPERTY STRINGS "CUDA" "HIP")
+
+if(${CUDAWRAPPERS_BACKEND} STREQUAL "CUDA")
+  set(CUDAWRAPPERS_BACKEND_CUDA True)
+elseif(${CUDAWRAPPERS_BACKEND} STREQUAL "HIP")
+  set(CUDAWRAPPERS_BACKEND_HIP True)
+else()
+  message(
+    FATAL_ERROR
+      "Invalid value for CUDAWRAPPERS_BACKEND: ${CUDAWRAPPERS_BACKEND}"
+  )
+endif()
+
+if(${CUDAWRAPPERS_BACKEND_HIP})
+  enable_language(HIP)
+  set(CMAKE_HIP_COMPILER_ENV_VAR "HIPCXX")
+endif()
+
 # Enable Clang compiler warnings
 if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
   add_compile_options(

diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@
 [![fair-software.eu](https://img.shields.io/badge/fair--software.eu-%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F-green)](https://fair-software.eu)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/d38b0338fda24733ab41a64915af8248)](https://app.codacy.com/gh/nlesc-recruit/cudawrappers/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=nlesc-recruit/cudawrappers&amp;utm_campaign=Badge_Grade)
 [![citation metadata](https://github.com/nlesc-recruit/cudawrappers/actions/workflows/cffconvert.yml/badge.svg)](https://github.com/nlesc-recruit/cudawrappers/actions/workflows/cffconvert.yml)
-[![Documentation Status](https://readthedocs.org/projects/cudawrappers/badge/?version=latest)](https://cudawrappers.readthedocs.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/cudawrappers/badge)](https://cudawrappers.readthedocs.io/en/latest/)
 
 
 # cudawrappers
@@ -17,20 +17,29 @@ This library is a C++ wrapper for the Nvidia C libraries (e.g. CUDA driver, nvrt
 2. _better fault handling_ (through exceptions);
 3. _more compact user code_.
 
+This library also supports AMD GPUs through the HIP: C++ Heterogeneous-Compute
+Interface for Portability.
+
 Originally, the API enforced RAII to even further reduce the risk of faulty code, but enforcing RAII and compatibility with (unmanaged) objects obtained outside this API are mutually exclusive.
 
 ## Requirements
 
 | Software    | Minimum version |
 | ----------- | ----------- |
 | CUDA        | 10.0 or later |
+| ROCM        | 6.1.0 or later |
 | CMake       | 3.17 or later |
 | gcc         | 9.3 or later  |
 | OS          | Linux distro (amd64) |
 
 | Hardware    | Type |
 | ----------- | ----------- |
-| GPU architecture        | [NVIDIA PASCAL](https://www.nvidia.com/en-in/geforce/products/10series/architecture/) or newer|
+| NVIDIA GPU  |
+[Pascal](https://www.nvidia.com/en-in/geforce/products/10series/architecture/)
+or newer|
+| AMD GPU     | RDNA2 or newer, CDNA2 or newer |
+
+
 
 ## Usage
 
@@ -55,6 +64,18 @@ make -C build
 make -C build install
 ```
 
+## Usage (HIP)
+
+To enable HIP, make sure to build cudawrappers with
+`-DCUDAWRAPPERS_BACKEND=HIP`, or when using `FetchContent`, use
+`set(CUDAWRAPPERS_BACKEND "HIP")`.  In your project's `CMakeLists.txt`, add
+`enable_language(HIP)`. Furthermore, every target that includes a cudawrappers
+header file needs to be 'hipified', to this end, add
+`set_source_files_properties(source.cpp PROPERTIES LANGUAGE HIP)` for every
+relevant `source.cpp` file. Some CUDA specific features may not be available or
+not work on non-NVIDIA GPUs, in those cases use `#ifdef(__HIP__)` guards to
+patch your code wherever this is needed.
+
 ### Usage examples
 
 You can include the cudawrappers library in your own projects in various ways. We have created a few repositories with example setups to get you started:

diff --git a/cmake/cudawrappers-config.cmake.in b/cmake/cudawrappers-config.cmake.in
@@ -2,4 +2,4 @@
 
 include(${PACKAGE_PREFIX_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/@PROJECT_NAME@/cudawrappers-dependencies.cmake)
 include(${PACKAGE_PREFIX_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/@PROJECT_NAME@/cudawrappers-exported.cmake)
-include(${PACKAGE_PREFIX_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/@PROJECT_NAME@/cudawrappers-helper.cmake)
+include(${PACKAGE_PREFIX_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/@PROJECT_NAME@/cudawrappers-helper.cmake)
diff --git a/cmake/cudawrappers-dependencies.cmake b/cmake/cudawrappers-dependencies.cmake
@@ -1,19 +1,77 @@
-# cudawrappers requires the CUDA Toolkit.If you include cudawrappers in your
-# project, you need to include the toolkit yourself
-set(CUDA_MIN_VERSION 10.0)
-if(${CUDAWRAPPERS_INSTALLED})
-  find_package(CUDAToolkit ${CUDA_MIN_VERSION} REQUIRED)
+if(${CUDAWRAPPERS_BACKEND_HIP})
+  # cmake-format: off
+  # This following code attempts to locate the HIP runtime library's root
+  # directory.
+  #
+  # 1. Checks if the ROCM_ROOT and ROCM_PATH variables are defined.
+  #    - Assign them from corresponding environment variables if not defined.
+  # 2. Searches for the HIP runtime header file in the paths specified
+  #    by ROCM_ROOT and ROCM_PATH.
+  #    - If the path is set but HIP is not found, an error message is generated.
+  # 3. If HIP is still not found, the script searches in the default path.
+  # 4. Adding HIP_ROOT_DIR to CMAKE_PREFIX_PATH and Finding Packages
+  #    - The HIP runtime directory is appended to CMAKE_PREFIX_PATH
+  #      to ensure that CMake can find the HIP-related packages.
+  #
+  # Usage:
+  # Set the ROCM_ROOT or ROCM_PATH environment variables, or
+  # pass them as CMake options, e.g.:
+  #   cmake -DROCM_ROOT=/path/to/rocm ..
+  #   or
+  #   ROCM_ROOT=/path/to/rocm cmake ..
+  # cmake-format: on
+
+  foreach(var IN ITEMS ROCM_ROOT ROCM_PATH)
+    # Step 1.
+    if(NOT DEFINED ${var})
+      set(${var} $ENV{${var}})
+    endif()
+
+    # Step 2.
+    find_path(
+      HIP_ROOT_DIR
+      NAMES include/hip/hip_runtime.h
+      PATHS ${${var}}
+      NO_DEFAULT_PATH
+    )
+
+    if(NOT HIP_ROOT_DIR AND ${var})
+      message(FATAL_ERROR "HIP not found in " ${${var}})
+    endif()
+  endforeach()
+
+  if(NOT HIP_ROOT_DIR)
+    # Step 3.
+    find_path(
+      HIP_ROOT_DIR
+      NAMES include/hip/hip_runtime.h
+      PATHS /opt/rocm
+    )
+  endif()
+
+  # Step 4.
+  list(APPEND CMAKE_PREFIX_PATH ${HIP_ROOT_DIR})
+  set(HIP_MIN_VERSION 6.1)
+  find_package(hip ${HIP_MIN_VERSION} REQUIRED)
+  find_package(hipfft REQUIRED)
 else()
-  if(${CUDAToolkit_FOUND})
-    if(${CUDAToolkit_VERSION_MAJOR} LESS ${CUDA_MIN_VERSION})
-      message(FATAL_ERROR "Insufficient CUDA version: " ${CUDAToolkit_VERSION}
-                          " < " ${CUDA_MIN_VERSION}
+  # cudawrappers requires the CUDA Toolkit.If you include cudawrappers in your
+  # project, you need to include the toolkit yourself
+  set(CUDA_MIN_VERSION 10.0)
+  if(${CUDAWRAPPERS_INSTALLED})
+    find_package(CUDAToolkit ${CUDA_MIN_VERSION} REQUIRED)
+  else()
+    if(${CUDAToolkit_FOUND})
+      if(${CUDAToolkit_VERSION_MAJOR} LESS ${CUDA_MIN_VERSION})
+        message(FATAL_ERROR "Insufficient CUDA version: "
+                            ${CUDAToolkit_VERSION} " < " ${CUDA_MIN_VERSION}
+        )
+      endif()
+    else()
+      message(
+        FATAL_ERROR
+          "CUDAToolkit not found, use find_package(CUDAToolkit REQUIRED)."
       )
     endif()
-  else()
-    message(
-      FATAL_ERROR
-        "CUDAToolkit not found, use find_package(CUDAToolkit REQUIRED)."
-    )
   endif()
 endif()
diff --git a/cmake/cudawrappers-targets.cmake b/cmake/cudawrappers-targets.cmake
@@ -5,11 +5,21 @@ get_filename_component(
 
 # Define all the individual components that cudawrappers provides
 set(CUDAWRAPPERS_COMPONENTS cu cufft nvml nvrtc nvtx)
-set(LINK_cu CUDA::cuda_driver)
-set(LINK_cufft CUDA::cuda_driver CUDA::cufft)
-set(LINK_nvml CUDA::cuda_driver CUDA::nvml)
-set(LINK_nvrtc CUDA::cuda_driver CUDA::nvrtc)
-set(LINK_nvtx CUDA::nvToolsExt)
+if(${CUDAWRAPPERS_BACKEND_HIP})
+  list(APPEND CUDAWRAPPERS_COMPONENTS macros)
+  set(LINK_macros hip::host)
+  set(LINK_cu hip::host)
+  set(LINK_cufft hip::host hip::hipfft)
+  set(LINK_nvml hip::host)
+  set(LINK_nvrtc hip::host)
+  set(LINK_nvtx hip::host)
+else()
+  set(LINK_cu CUDA::cuda_driver)
+  set(LINK_cufft CUDA::cuda_driver CUDA::cufft)
+  set(LINK_nvml CUDA::cuda_driver CUDA::nvml)
+  set(LINK_nvrtc CUDA::cuda_driver CUDA::nvrtc)
+  set(LINK_nvtx CUDA::nvToolsExt)
+endif()
 
 foreach(component ${CUDAWRAPPERS_COMPONENTS})
   add_library(${component} INTERFACE)