Refact/gko 180 reuse pgm

.github/workflows/build-foam.yml

@@ -58,7 +58,7 @@ jobs:
     - name: Get Ginkgo checkout version
       shell: bash
       run: |
-        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
+        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-_.]*" > GINKGO_CHECKOUT_VERSION
         export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
         echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
 
@@ -86,7 +86,11 @@ jobs:
         mkdir -p build
         cd build
         [ -d "/github/home/$GINKGO_CHECKOUT_VERSION" ] && cp -rp /github/home/$GINKGO_CHECKOUT_VERSION third_party
-        cmake -GNinja -DOGL_ALLOW_REFERENCE_ONLY=On -DOGL_BUILD_UNITTEST=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=${{matrix.OF.foam_user_libbin}} ..
+        cmake -GNinja \
+          -DOGL_ALLOW_REFERENCE_ONLY=On \
+          -DOGL_BUILD_UNITTEST=ON \
+          -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+          -DCMAKE_INSTALL_PREFIX=${{matrix.OF.foam_user_libbin}} ..
 
     - name: Build OGL
       working-directory: ${{github.workspace}}/build

.github/workflows/static_checks.yaml

@@ -5,36 +5,41 @@
 name: Static checks
 run-name: Static checks
 
+defaults:
+ run:
+  shell: bash -o pipefail -i {0}
+
 on:
     pull_request:
       types: [opened, synchronize]
+
+env:
+  USER: user
+  BUILD_TYPE: Release
+  OMPI_ALLOW_RUN_AS_ROOT: 1
+  OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+
+
 jobs:
   build-compilation-db:
     name: Build with IWYU
     runs-on: ubuntu-latest
+    container: greole/ofbase
     steps:
     - uses: actions/checkout@v4
-
-    - name: Add clang repo
-      run: |
-        sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main'
-        wget https://apt.llvm.org/llvm-snapshot.gpg.key
-        sudo apt-key add llvm-snapshot.gpg.key
-
     - name: Install dependencies
       run: |
-        sudo apt update
-        sudo apt install \
+        apt update
+        apt install -y \
            ninja-build \
-           iwyu \
-           clang-16 \
-           libomp-16-dev
-
+           libomp-dev \
+           iwyu
     - name: Create Compilation Database
       run: |
+        source /root/OpenFOAM/openfoam/etc/bashrc
         cmake --preset ninja-cpuonly-release \
-        -DCMAKE_CXX_COMPILER=clang++-16 \
-        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+          -DCMAKE_CXX_COMPILER=clang++ \
+          -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
         cmake --build --preset ninja-cpuonly-release
     - uses: actions/upload-artifact@v4
       with:
@@ -43,40 +48,33 @@ jobs:
   clang-tidy-check:
     name: Clang-tidy Check
     runs-on: ubuntu-latest
+    container: greole/ofbase
     needs: [build-compilation-db]
     steps:
     - uses: actions/checkout@v4
-
-    - name: Add clang repo
-      run: |
-        sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main'
-        wget https://apt.llvm.org/llvm-snapshot.gpg.key
-        sudo apt-key add llvm-snapshot.gpg.key
-
     - name: Install dependencies
       run: |
-        sudo apt update
-        sudo apt install \
-           clang-tidy-16 \
-           libomp-16-dev
-
+        apt update
+        apt install -y \
+           clang-tidy \
+           jq
     - uses: actions/download-artifact@v4
       with:
         name: build-artifact
         path: build
-
     - name: Run clang-tidy
       run: |
-        # Create list of .cpp files belonging to this repository
-        git ls-files | grep -E "\.(cpp)" > pattern
+        git config --global --add safe.directory /__w/OGL/OGL
+        # Create list of all source files belonging to this repository
+        git ls-files | grep -E "\.(C)" > pattern
         # Create list of .cpp files that are in this repository and part of the
         # compilation database
         # also filters out " at the begin and end of the filename
-        jq ".[] | .file" build/ReleaseAll/compile_commands.json \
+        jq ".[] | .file" build/ReleaseCPUOnly/compile_commands.json \
           | sed 's/^"\(.*\)"$/\1/' \
           | grep -F -f pattern - > files
         # run clang-tidy on all specified files
-        clang-tidy-16 --fix --extra-arg=-w -p build/ReleaseAll $(cat files)
+        clang-tidy --fix --extra-arg=-w -p build/ReleaseCPUOnly $(cat files)
 
     - name: Check for fixes
       run: |

CHANGELOG.md

@@ -3,7 +3,8 @@ SPDX-FileCopyrightText: 2024 OGL authors
 
 SPDX-License-Identifier: GPL-3.0-or-later
 -->
-
+# 0.6.0 (unreleased)
+- Build Ginkgo as subdirectory build [PR #135](https://github.com/hpsim/OGL/pull/135)
 # 0.5.4 (unreleased)
 - Add a warning message to users who use dpcpp as the executor name [PR #129](https://github.com/hpsim/OGL/pull/129)
 - Add support for SP scalars and determine label and scalar size from env variable [PR #120](https://github.com/hpsim/OGL/pull/120) 

CMakeLists.txt

@@ -14,8 +14,11 @@ if(NOT DEFINED ENV{FOAM_SRC})
   message(FATAL_ERROR "You must source OpenFOAM before building OGL")
 endif()
 
-# Configuration options
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(BUILD_SHARED_LIBS ON)
 
+# Configuration options
 include(cmake/build_type_helpers.cmake)
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
 
@@ -26,16 +29,37 @@ option(OGL_USE_FOAM_FOUNDATION_VERSION
        "Build OGL for the OpenFOAM Foundation version" FALSE)
 option(OGL_BUILD_UNITTEST "Build OGL unittests" FALSE)
 option(OGL_CUDA_ARCHITECTURES "Which cuda architecture to build for" "native")
-option(GINKGO_BUILD_CUDA "Build Ginkgo with cuda backend" FALSE)
-option(GINKGO_BUILD_HIP "Build Ginkgo with hip backend" FALSE)
-option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE)
-option(GINKGO_BUILD_OMP "Build Ginkgo with omp backend" FALSE)
 option(GINKGO_BUILD_REFERENCE "Build Ginkgo with reference backend" TRUE)
 option(OGL_ALLOW_REFERENCE_ONLY
        "Enable builds of Ginkgo with reference backend only" FALSE)
-option(GINKGO_FORCE_GPU_AWARE_MPI "Build Ginkgo using device aware MPI" TRUE)
+option(OGL_ENABLE_SANITIZE_ADDRESS "Enable address sanitizer" FALSE)
+option(OGL_ENABLE_SANITIZE_LEAK "Enable leak sanitizer" FALSE)
+option(OGL_ENABLE_SANITIZE_UB "Enable undefined behaviour sanitizer" FALSE)
+option(OGL_ENABLE_SANITIZE_THREAD "Enable thread sanitizer" FALSE)
+option(OGL_ENABLE_SANITIZE_MEMORY "Enable memory sanitizer" FALSE)
+
 option(GINKGO_WITH_OGL_EXTENSIONS "Whether ginkgo was build with OGL extension"
        FALSE)
+option(GINKGO_BUILD_REFERENCE "Build Ginkgo with reference backend" TRUE)
+
+option(GINKGO_FORCE_GPU_AWARE_MPI "Build Ginkgo using device aware MPI" TRUE)
+option(GINKGO_BUILD_CUDA "Build Ginkgo with cuda backend" FALSE)
+option(GINKGO_BUILD_HIP "Build Ginkgo with hip backend" FALSE)
+option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE)
+option(GINKGO_BUILD_OMP "Build Ginkgo with omp backend" FALSE)
+
+set(GINKGO_BUILD_MPI
+    ON
+    CACHE INTERNAL "")
+set(GINKGO_BUILD_TESTS
+    OFF
+    CACHE INTERNAL "")
+set(GINKGO_BUILD_EXAMPLES
+    OFF
+    CACHE INTERNAL "")
+set(GINKGO_BUILD_BENCHMARKS
+    OFF
+    CACHE INTERNAL "")
 
 if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO))
   if((NOT GINKGO_BUILD_CUDA)
@@ -49,7 +73,7 @@ if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO))
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "fc86d48b78cebd2b2c5833a2dcf0fe40f615cf19"
+    "reuse_pgm"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)
@@ -58,44 +82,24 @@ check_include_file_cxx(cxxabi.h GKO_HAVE_CXXABI_H)
 include(cmake/package_helpers.cmake)
 include(cmake/build_helpers.cmake)
 include(cmake/install_helpers.cmake)
+include(cmake/Sanitizer.cmake)
+include(cmake/OpenFOAM.cmake)
 
-ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.7.0)
-
-add_subdirectory(third_party)
-
-# If ginkgo is not found at this point, look for the third_party installation
-if(NOT ${OGL_USE_EXTERNAL_GINKGO})
-  find_package(
-    Ginkgo
-    CONFIG
-    1.7.0
-    REQUIRED
-    PATHS
-    $ENV{FOAM_USER_LIBBIN}/cmake/Ginkgo
-    ${CMAKE_CURRENT_BINARY_DIR}/third_party/ginkgo/build/install/lib/cmake)
-endif()
-
-# C++ 14 standard or later is required to interface with Ginkgo OGL itself uses
-# some c++ 17 features
-
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.8.0)
+include(cmake/ginkgo.cmake)
 
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0 -ggdb")
 set(CMAKE_CXX_FLAGS_RELEASE
     "${CMAKE_CXX_FLAGS_RELEASE} \
-  -Wall -Wpedantic -Wextra -march=native -fopenmp -fPIC -Wno-undefined-var-template"
+  -Wall -Wpedantic -Wextra -march=native -fopenmp -Wno-undefined-var-template"
 )
 
 if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
   message("Compiling with clang")
   set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-  set(CMAKE_CXX_FLAGS_DEBUG
-      "${CMAKE_CXX_FLAGS_DEBUG} \
-    -fno-omit-frame-pointer \
-    -fsanitize=address")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} \
+    -fno-omit-frame-pointer")
 
   set(CMAKE_CXX_FLAGS_RELEASE
       "${CMAKE_CXX_FLAGS_RELEASE} \
@@ -113,8 +117,8 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
       "${CMAKE_CXX_FLAGS_DEBUG} \
       --coverage\
       -fprofile-arcs \
-      -ggdb3")
-  # * fsanitize = address \ - ftest - coverage \
+      -ggdb3 \
+      ")
 
   set(CMAKE_CXX_FLAGS_RELEASE
       "${CMAKE_CXX_FLAGS_RELEASE} \
@@ -125,7 +129,6 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
       ")
 endif()
 
-option(BUILD_SHARED_LIBS "Build shared library" ON)
 add_library(OGL "")
 
 target_compile_definitions(OGL PUBLIC NoRepository)
@@ -138,6 +141,7 @@ if(APPLE)
   target_compile_definitions(OGL PRIVATE APPLE arm64)
 else()
   target_compile_definitions(OGL PRIVATE linux64)
+  target_link_libraries(OGL PRIVATE stdc++fs)
 endif()
 
 # Add a custom command that produces version.cpp, plus a dummy output that's not
@@ -154,57 +158,49 @@ target_sources(
   PRIVATE common/common.C
           ${CMAKE_CURRENT_BINARY_DIR}/version.C
           lduLduBase/lduLduBase.C
+          HostMatrix/HostMatrix.C
+          HostMatrix/HostMatrixFreeFunctions.C
           StoppingCriterion/StoppingCriterion.C
           DevicePersistent/Base/Base.C
           DevicePersistent/Partition/Partition.C
           DevicePersistent/Array/Array.C
           DevicePersistent/Vector/Vector.C
           DevicePersistent/DeviceIdGuard/DeviceIdGuard.C
           DevicePersistent/ExecutorHandler/ExecutorHandler.C
-          DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.C
           Preconditioner/Preconditioner.C
           BaseWrapper/lduBase/GKOlduBase.C
           BaseWrapper/CoupledLduBase/GKOCoupledLduBase.C
-          HostMatrix/HostMatrix.C
-          HostMatrix/HostMatrixFreeFunctions.C
           Solver/CG/GKOCG.C
           Solver/BiCGStab/GKOBiCGStab.C
-          # Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C
+          # Solver / IR / GKOIR.C
+          Solver/Multigrid/GKOMultigrid.C
           Solver/GMRES/GKOGMRES.C
-  # LduMatrix / GKOACG / GKOACG.C
   PUBLIC common/common.H
          StoppingCriterion/StoppingCriterion.H
          lduLduBase/lduLduBase.H
-         HostMatrix/HostMatrix.H
          DevicePersistent/Base/Base.H
          DevicePersistent/Partition/Partition.H
          DevicePersistent/Array/Array.H
          DevicePersistent/Vector/Vector.H
          DevicePersistent/ExecutorHandler/ExecutorHandler.H
          DevicePersistent/DeviceIdGuard/DeviceIdGuard.H
-         DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
          Preconditioner/Preconditioner.H
          BaseWrapper/lduBase/GKOlduBase.H
          BaseWrapper/CoupledLduBase/GKOCoupledLduBase.H
          Solver/CG/GKOCG.H
          Solver/IR/GKOIR.H
-         # Solver / Multigrid / GKOMultigrid.H
          Solver/BiCGStab/GKOBiCGStab.H
          Solver/GMRES/GKOGMRES.H
-  # LduMatrix / GKOACG / GKOACG.H
 )
 
-target_include_directories(
-  OGL SYSTEM
-  PUBLIC $ENV{FOAM_SRC}/finiteVolume/lnInclude
-         $ENV{FOAM_SRC}/meshTools/lnInclude $ENV{FOAM_SRC}/OpenFOAM/lnInclude
-         $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude ${CMAKE_CURRENT_SOURCE_DIR})
+enable_sanitizers(
+  OGL ${OGL_ENABLE_SANITIZE_ADDRESS} ${OGL_ENABLE_SANITIZE_LEAK}
+  ${OGL_ENABLE_SANITIZE_UB} ${OGL_ENABLE_SANITIZE_THREAD}
+  ${OGL_ENABLE_SANITIZE_MEMORY})
 
-target_link_libraries(
-  OGL
-  PRIVATE $ENV{FOAM_LIBBIN}/libOpenFOAM.so $ENV{FOAM_LIBBIN}/libfiniteVolume.so
-          $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so
-  PUBLIC Ginkgo::ginkgo stdc++fs)
+target_include_directories(OGL SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+
+target_link_libraries(OGL PRIVATE OpenFOAM Ginkgo::ginkgo)
 
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)
@@ -236,4 +232,13 @@ add_custom_target(
   COMMAND find . -name CMakeLists.txt -exec cmake-format -i {} +
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
 
-install(TARGETS OGL DESTINATION $ENV{FOAM_USER_LIBBIN})
+install(
+  TARGETS OGL
+          ginkgo
+          ginkgo_device
+          ginkgo_hip
+          ginkgo_cuda
+          ginkgo_omp
+          ginkgo_dpcpp
+          ginkgo_reference
+  DESTINATION $ENV{FOAM_USER_LIBBIN})