Improve CMake

CMakeLists.txt

@@ -8,16 +8,10 @@ INCLUDE(SetupTPLs)
 INCLUDE(SetupMFMG)
 INCLUDE(SetupCache)
 
-# By default build a static library because CUDA does not work with a shared
-# library
-OPTION(BUILD_SHARED_LIBS "Build a shared library" OFF)
 
 INITIALIZE_CACHE()
 
 IF(${MFMG_ENABLE_CUDA})
-  IF (${BUILD_SHARED_LIBS})
-    MESSAGE(SEND_ERROR "CUDA build does not support shared library")
-  ENDIF()
   # ENABLE_LANGUAGE cannot be in a function and it has to be called in the
   # highest directory so we cannot encapsulate it
   ENABLE_LANGUAGE(CUDA)
@@ -74,7 +68,7 @@ TARGET_INCLUDE_DIRECTORIES(mfmg SYSTEM PUBLIC ${DEAL_II_INCLUDE_DIRS})
 TARGET_LINK_LIBRARIES(mfmg PUBLIC ${DEAL_II_LIBRARIES})
 TARGET_LINK_LIBRARIES(mfmg PUBLIC ${MFMG_CUDA_LIBRARIES})
 TARGET_INCLUDE_DIRECTORIES(mfmg PUBLIC ${AMGX_INCLUDE_DIR})
-TARGET_LINK_LIBRARIES(mfmg PUBLIC ${AMGX_LIBRARY})
+TARGET_LINK_LIBRARIES(mfmg PUBLIC ${AMGX_LIBRARIES})
 TARGET_COMPILE_DEFINITIONS(mfmg PRIVATE ${CLANG_TIDY_DEFINITIONS})
 
 IF(${Backtrace_FOUND})

ci/compile_and_run.sh

@@ -1,8 +1,20 @@
 #!/bin/bash
+# In Docker containers and for some installations, stub libraries are installed.
+# The CUDA libraries are installed with the driver but to allow compilation on
+# system without the correct drivers stub libraries are installed when the CUDA
+# toolkit is installed. The stub libraries allow you to compile the code but not
+# to run it because the symbol are missing. The idea is that at run time, we will
+# pick the correct libraries instead of the stub ones. This doesn't work when
+# using an `rpath` (see https://github.com/NVIDIA/nvidia-docker/issues/775). For
+# some reason the problem only appears with CUSolver having missing symbol to
+# OpenMP. Removing the stubs fixes the problem.
+#rm -r  /usr/local/cuda/lib64/stubs
 cd $1
 rm -rf build
+export LD_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/5.4.0:${LD_LIBRARY_PATH}
 mkdir build && cd build
 ARGS=(
+  -D BUILD_SHARED_LIBS=OFF
   -D CMAKE_BUILD_TYPE=Debug
   -D MFMG_ENABLE_TESTS=ON
   -D MFMG_ENABLE_CUDA=ON

cmake/SetupCUDA.cmake

@@ -2,23 +2,39 @@
 # the list of CUDA libraries that we are using
 FUNCTION(SET_CUDA_LIBRARIES)
   ADD_DEFINITIONS(-DMFMG_WITH_CUDA)
+  FIND_LIBRARY(CUSPARSE cusparse PATH ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+  FIND_LIBRARY(CUSOLVER cusolver PATH ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+  MESSAGE("implicit ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}")
   SET(MFMG_CUDA_LIBRARIES
-    "cusparse"
-    "cusolver"
+    ${CUSPARSE}
+    ${CUSOLVER}
     PARENT_SCOPE
     )
-  # cuSolver needs OpenMP
+
+#### OPENMP ##################################################################
+# cuSOLVER needs OpenMP
   FIND_PACKAGE(OpenMP)
   IF(OPENMP_FOUND)
     SET(CMAKE_CXX_FLAGS
       "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}"
       PARENT_SCOPE
       )
-    SET(CMAKE_EXE_LINKER_FLAGS
-      "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}"
-      PARENT_SCOPE
-      )
   ELSE()
     MESSAGE(SEND_ERROR "Could not find OpenMP required by cuSolver")
   ENDIF()
+
+#### AMGX ####################################################################
+  IF(${MFMG_ENABLE_CUDA})
+    IF (${MFMG_ENABLE_AMGX})
+      FIND_LIBRARY(AMGX_LIBRARY amgx amgxsh PATH "${AMGX_DIR}/lib")
+      FIND_LIBRARY(CUBLAS cublas HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+      SET(AMGX_LIBRARIES
+        ${AMGX_LIBRARY}
+        ${CUBLAS}
+        PARENT_SCOPE
+        )
+      SET(AMGX_INCLUDE_DIR "${AMGX_DIR}/include" PARENT_SCOPE)
+      ADD_DEFINITIONS(-DMFMG_WITH_AMGX)
+    ENDIF()
+  ENDIF()
 ENDFUNCTION()

cmake/SetupTPLs.cmake

@@ -32,16 +32,3 @@ ELSE()
     ADD_DEFINITIONS(-DDEBUG)
   ENDIF()
 ENDIF()
-
-#### AMGX ####################################################################
-IF(${MFMG_ENABLE_CUDA})
-  IF (${MFMG_ENABLE_AMGX})
-    ADD_LIBRARY(amgx STATIC IMPORTED)
-    SET_TARGET_PROPERTIES(amgx PROPERTIES IMPORTED_LOCATION
-      "${AMGX_DIR}/lib/libamgx.a"
-      IMPORTED_LINK_INTERFACE_LIBRARIES "cublas")
-    ADD_DEFINITIONS(-DMFMG_WITH_AMGX)
-    SET(AMGX_INCLUDE_DIR "${AMGX_DIR}/include")
-    SET(AMGX_LIBRARY amgx)
-  ENDIF()
-ENDIF()

include/mfmg/dealii_operator_device.cuh

@@ -150,6 +150,13 @@ private:
   AMGX_solver_handle _amgx_solver_handle;
   int _device_id[1];
   std::unordered_map<int, int> _row_map;
+#else
+  void *_amgx_config_handle;
+  void *_amgx_res_handle;
+  void *_amgx_matrix_handle;
+  void *_amgx_rhs_handle;
+  void *_amgx_solution_handle;
+  void *_amgx_solver_handle;
 #endif
 };
 } // namespace mfmg

tests/CMakeLists.txt

@@ -23,7 +23,9 @@ IF(${MFMG_ENABLE_CUDA})
   MFMG_ADD_CUDA_TEST(test_sparse_matrix_device_operator 1)
   MFMG_ADD_CUDA_TEST(test_smoother_device 1)
   MFMG_ADD_CUDA_TEST(test_hierarchy_device 1)
-  MFMG_ADD_CUDA_TEST(test_amgx_direct_solver 1 2)
+  IF(${MFMG_ENABLE_AMGX})
+    MFMG_ADD_CUDA_TEST(test_amgx_direct_solver 1 2)
+  ENDIF()
 ENDIF()
 
 MFMG_COPY_INPUT_FILE(hierarchy_input.info  tests/data)