From 1e5ad1332c8267354c8f63743329383ad6034556 Mon Sep 17 00:00:00 2001
From: Bruno Turcksin <bruno.turcksin@gmail.com>
Date: Tue, 28 Aug 2018 16:45:59 -0400
Subject: [PATCH] Update build system to allow shared library when using CUDA

---
 CMakeLists.txt        |  8 +-------
 ci/compile_and_run.sh | 11 +++++++++++
 cmake/SetupCUDA.cmake | 30 +++++++++++++++++++++++-------
 cmake/SetupTPLs.cmake | 13 -------------
 4 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a528fc51..41b531ca 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,16 +8,10 @@ INCLUDE(SetupTPLs)
 INCLUDE(SetupMFMG)
 INCLUDE(SetupCache)
 
-# By default build a static library because CUDA does not work with a shared
-# library
-OPTION(BUILD_SHARED_LIBS "Build a shared library" OFF)
 
 INITIALIZE_CACHE()
 
 IF(${MFMG_ENABLE_CUDA})
-  IF (${BUILD_SHARED_LIBS})
-    MESSAGE(SEND_ERROR "CUDA build does not support shared library")
-  ENDIF()
   # ENABLE_LANGUAGE cannot be in a function and it has to be called in the
   # highest directory so we cannot encapsulate it
   ENABLE_LANGUAGE(CUDA)
@@ -74,7 +68,7 @@ TARGET_INCLUDE_DIRECTORIES(mfmg SYSTEM PUBLIC ${DEAL_II_INCLUDE_DIRS})
 TARGET_LINK_LIBRARIES(mfmg PUBLIC ${DEAL_II_LIBRARIES})
 TARGET_LINK_LIBRARIES(mfmg PUBLIC ${MFMG_CUDA_LIBRARIES})
 TARGET_INCLUDE_DIRECTORIES(mfmg PUBLIC ${AMGX_INCLUDE_DIR})
-TARGET_LINK_LIBRARIES(mfmg PUBLIC ${AMGX_LIBRARY})
+TARGET_LINK_LIBRARIES(mfmg PUBLIC ${AMGX_LIBRARIES})
 TARGET_COMPILE_DEFINITIONS(mfmg PRIVATE ${CLANG_TIDY_DEFINITIONS})
 
 IF(${Backtrace_FOUND})
diff --git a/ci/compile_and_run.sh b/ci/compile_and_run.sh
index 7c7a198d..62e4d496 100755
--- a/ci/compile_and_run.sh
+++ b/ci/compile_and_run.sh
@@ -1,6 +1,17 @@
 #!/bin/bash
+# In Docker containers and for some installations, stub libraries are installed.
+# The CUDA libraries are installed with the driver but to allow compilation on
+# system without the correct drivers stub libraries are installed when the CUDA
+# toolkit is installed. The stub libraries allow you to compile the code but not
+# to run it because the symbol are missing. The idea is that at run time, we will
+# pick the correct libraries instead of the stub ones. This doesn't work when
+# using an `rpath` (see https://github.com/NVIDIA/nvidia-docker/issues/775). For
+# some reason the problem only appears with CUSolver having missing symbol to
+# OpenMP. Removing the stubs fixes the problem.
+rm -r  /usr/local/cuda/lib64/stubs
 cd $1
 rm -rf build
+export LD_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/5.4.0:${LD_LIBRARY_PATH}
 mkdir build && cd build
 ARGS=(
   -D CMAKE_BUILD_TYPE=Debug
diff --git a/cmake/SetupCUDA.cmake b/cmake/SetupCUDA.cmake
index 1db1862d..b4aaf46f 100644
--- a/cmake/SetupCUDA.cmake
+++ b/cmake/SetupCUDA.cmake
@@ -2,23 +2,39 @@
 # the list of CUDA libraries that we are using
 FUNCTION(SET_CUDA_LIBRARIES)
   ADD_DEFINITIONS(-DMFMG_WITH_CUDA)
+  FIND_LIBRARY(CUSPARSE cusparse HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+  FIND_LIBRARY(CUSOLVER cusolver HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+  MESSAGE("implicit ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}")
   SET(MFMG_CUDA_LIBRARIES
-    "cusparse"
-    "cusolver"
+    ${CUSPARSE}
+    ${CUSOLVER}
     PARENT_SCOPE
     )
-  # cuSolver needs OpenMP
+
+#### OPENMP ##################################################################
+# cuSOLVER needs OpenMP
   FIND_PACKAGE(OpenMP)
   IF(OPENMP_FOUND)
     SET(CMAKE_CXX_FLAGS
       "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}"
       PARENT_SCOPE
       )
-    SET(CMAKE_EXE_LINKER_FLAGS
-      "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}"
-      PARENT_SCOPE
-      )
   ELSE()
     MESSAGE(SEND_ERROR "Could not find OpenMP required by cuSolver")
   ENDIF()
+
+#### AMGX ####################################################################
+  IF(${MFMG_ENABLE_CUDA})
+    IF (${MFMG_ENABLE_AMGX})
+      FIND_LIBRARY(AMGX_LIBRARY amgx amgxsh PATH "${AMGX_DIR}/lib")
+      FIND_LIBRARY(CUBLAS cublas HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+      SET(AMGX_LIBRARIES
+        ${AMGX_LIBRARY}
+        ${CUBLAS}
+        PARENT_SCOPE
+        )
+      SET(AMGX_INCLUDE_DIR "${AMGX_DIR}/include" PARENT_SCOPE)
+      ADD_DEFINITIONS(-DMFMG_WITH_AMGX)
+    ENDIF()
+  ENDIF()
 ENDFUNCTION()
diff --git a/cmake/SetupTPLs.cmake b/cmake/SetupTPLs.cmake
index 515586a9..7bfa7a82 100644
--- a/cmake/SetupTPLs.cmake
+++ b/cmake/SetupTPLs.cmake
@@ -32,16 +32,3 @@ ELSE()
     ADD_DEFINITIONS(-DDEBUG)
   ENDIF()
 ENDIF()
-
-#### AMGX ####################################################################
-IF(${MFMG_ENABLE_CUDA})
-  IF (${MFMG_ENABLE_AMGX})
-    ADD_LIBRARY(amgx STATIC IMPORTED)
-    SET_TARGET_PROPERTIES(amgx PROPERTIES IMPORTED_LOCATION
-      "${AMGX_DIR}/lib/libamgx.a"
-      IMPORTED_LINK_INTERFACE_LIBRARIES "cublas")
-    ADD_DEFINITIONS(-DMFMG_WITH_AMGX)
-    SET(AMGX_INCLUDE_DIR "${AMGX_DIR}/include")
-    SET(AMGX_LIBRARY amgx)
-  ENDIF()
-ENDIF()