OMP target implementation

CMakeLists.txt

@@ -1,60 +1,15 @@
-cmake_minimum_required(VERSION 3.17 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 
-project(cloverleaf_sycl)
+project(cloverleaf_openmp_target)
 
 
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-#
-#set (OpenMP_CXX_FLAGS "-fiopenmp -qnextgen")
-#set (OpenMP_CXX_LIB_NAMES "libiomp5")
-#set (OpenMP_libiomp5_LIBRARY
-#        /opt/intel/oneapi/compiler/2021.1-beta08/linux/compiler/lib/intel64_lin/libiomp5.so
-#        )
 find_package(OpenMP REQUIRED)
 
 set(CMAKE_VERBOSE_MAKEFILE YES)
 
-#set(SYCL_RUNTIME DPCPP)
-#set(HIPSYCL_INSTALL_DIR /home/tom/hypsycl_dist)
-
-
-#if (SYCL_RUNTIME)
-#
-#    list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules)
-#
-#    if (${SYCL_RUNTIME} STREQUAL "HIPSYCL")
-#
-#        if (NOT HIPSYCL_INSTALL_DIR)
-#            message(FATAL_ERROR "HIPSYCL_INSTALL_DIR is undefined")
-#        endif ()
-#
-#        set(hipSYCL_DIR ${HIPSYCL_INSTALL_DIR}/lib/cmake)
-#        find_package(hipSYCL CONFIG REQUIRED)
-#        set(EXTRA_FLAGS -Wno-sign-compare -Wno-stringop-truncation)
-#    elseif (${SYCL_RUNTIME} STREQUAL "COMPUTECPP")
-#
-#        if (NOT ComputeCpp_DIR)
-#            message(FATAL_ERROR "ComputeCpp_DIR is undefined")
-#        endif ()
-#        add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
-#        set(COMPUTECPP_USER_FLAGS -O3 -fsycl-split-modules=20 -mllvm -inline-threshold=10000 -no-serial-memop)
-#        find_package(ComputeCpp REQUIRED)
-#        #        set(EXTRA_FLAGS  -pedantic)
-#    elseif (${SYCL_RUNTIME} STREQUAL "DPCPP")
-#
-#        set(CMAKE_CXX_STANDARD 17)
-#        set(CMAKE_CXX_COMPILER "dpcpp")
-#        set(EXTRA_FLAGS -pedantic)
-#        include_directories(/opt/intel/oneapi/compiler/2021.1-beta08/linux/compiler/include/)
-#    else ()
-#        message(FATAL_ERROR "SYCL_RUNTIME unsupported, must be one of HIPSYCL|COMPUTECPP|DPCPP, got ${SYCL_RUNTIME}")
-#    endif ()
-#else ()
-#    message(FATAL_ERROR "SYCL_RUNTIME not defined, must be one of HIPSYCL|COMPUTECPP|DPCPP")
-#endif ()
-
 
 if (MPI_AS_LIBRARY)
 
@@ -87,6 +42,7 @@ set(SOURCES
         src/advec_mom.cpp
         src/advection.cpp
         src/build_field.cpp
+        src/finalise_field.cpp
         src/calc_dt.cpp
         src/clover_leaf.cpp
         src/comms.cpp
@@ -113,9 +69,9 @@ set(SOURCES
         src/visit.cpp)
 
 include_directories(src)
-add_executable(clover_leaf ${SOURCES})
+add_executable(cloverleaf ${SOURCES})
 
-target_compile_options(clover_leaf
+target_compile_options(cloverleaf
         PUBLIC
         -Wall
         -Wextra
@@ -129,25 +85,42 @@ target_compile_options(clover_leaf
         ${EXTRA_FLAGS}
         )
 
-set(OMP_OFFLOAD_FLAGS -foffload=nvptx-none)
+separate_arguments(OMP_OFFLOAD_FLAGS)
+separate_arguments(CXX_EXTRA_FLAGS)
+separate_arguments(CXX_EXTRA_LINKER_FLAGS)
+
+
+option(OMP_ALLOW_HOST "Whether the OMP clause `if(target: <bool>)` is included at compile time, some compilers may not support this feature" ON)
+
+if (OMP_ALLOW_HOST)
+    add_definitions(-DOMP_ALLOW_HOST)
+endif()
+
+set(DEBUG_OPTIONS -O2 -fno-omit-frame-pointer ${OMP_OFFLOAD_FLAGS} ${CXX_EXTRA_FLAGS})
+set(RELEASE_OPTIONS -O3 ${OMP_OFFLOAD_FLAGS} ${CXX_EXTRA_FLAGS})
 
-set(DEBUG_OPTIONS -O2 -fno-omit-frame-pointer -fsanitize=address ${OMP_OFFLOAD_FLAGS} ${CXX_EXTRA_FLAGS})
-set(RELEASE_OPTIONS -Ofast -march=native -mtune=native ${OMP_OFFLOAD_FLAGS} ${CXX_EXTRA_FLAGS})
+target_link_libraries(cloverleaf PUBLIC ${MPI_C_LIB})
+target_link_libraries(cloverleaf PUBLIC OpenMP::OpenMP_CXX OpenMP::OpenMP_C)
 
-target_link_libraries(clover_leaf PUBLIC ${MPI_C_LIB})
-target_link_libraries(clover_leaf PUBLIC OpenMP::OpenMP_CXX OpenMP::OpenMP_C)
 
-# remove when using omp target
-#target_link_libraries(clover_leaf PUBLIC $<$<CONFIG:RelWithDebInfo>:-Wl,-lasan>)
-#target_link_libraries(clover_leaf PUBLIC $<$<CONFIG:Release>:-Wl,-lasan>)
-target_link_libraries(clover_leaf PUBLIC $<$<CONFIG:Debug>:-Wl,-lasan>)
+target_compile_options(cloverleaf PUBLIC "$<$<CONFIG:RelWithDebInfo>:${RELEASE_OPTIONS}>")
+target_compile_options(cloverleaf PUBLIC "$<$<CONFIG:Release>:${RELEASE_OPTIONS}>")
+target_compile_options(cloverleaf PUBLIC "$<$<CONFIG:Debug>:${DEBUG_OPTIONS}>")
 
-target_compile_options(clover_leaf PUBLIC "$<$<CONFIG:RelWithDebInfo>:${RELEASE_OPTIONS}>")
-target_compile_options(clover_leaf PUBLIC "$<$<CONFIG:Release>:${RELEASE_OPTIONS}>")
-target_compile_options(clover_leaf PUBLIC "$<$<CONFIG:Debug>:${DEBUG_OPTIONS}>")
 
+if (${CMAKE_VERSION} VERSION_LESS "3.13.0")
+    message(WARNING "target_link_options is only available in CMake >= 3.13.0, using fallback target_link_libraries, this may cause issues with some compilers")
 
-target_link_options(clover_leaf PUBLIC ${OpenMP_CXX_FLAGS} ${OMP_OFFLOAD_FLAGS})
+    set(EXTRA_LINK_FLAGS ${OpenMP_CXX_FLAGS} ${OMP_OFFLOAD_FLAGS})
 
-#target_link_options(clover_leaf PUBLIC LINKER:${CXX_EXTRA_LINKER_FLAGS})
+    if (DEFINED CXX_EXTRA_LINKER_FLAGS)
+        list(APPEND EXTRA_LINK_FLAGS "-Wl,${CXX_EXTRA_LINKER_FLAGS}")
+    endif ()
+
+    target_link_libraries(cloverleaf PUBLIC ${EXTRA_LINK_FLAGS})
+
+else ()
+    target_link_options(cloverleaf PUBLIC ${OpenMP_CXX_FLAGS} ${OMP_OFFLOAD_FLAGS})
+    target_link_options(cloverleaf PUBLIC LINKER:${CXX_EXTRA_LINKER_FLAGS})
+endif ()
 

README.md

@@ -28,28 +28,41 @@ Flags:
    * Set `MPI_C_INCLUDE_DIR` to  <mpi_root_dir>/include
    * Set `MPI_C_LIB` to the library name, for exampe: mpich for libmpich.so
  * `CXX_EXTRA_FLAGS` - `STRING`, appends extra flags that will be passed on to the compiler, applies to all configs
-  * `CXX_EXTRA_LINKER_FLAGS` - `STRING`, appends extra linker flags (the comma separated list after the `-Wl` flag) to the linker, applies to all configs
-
+ * `CXX_EXTRA_LINKER_FLAGS` - `STRING`, appends extra linker flags (the comma separated list after the `-Wl` flag) to the linker, applies to all configs
+ * `OMP_OFFLOAD_FLAGS` - OpenMP 4.5 target offload flags that will passed directly to the compiler and linker, see examples flag combinations below.
+    * GCC+NVIDIA - `"-foffload=nvptx-none -foffload=-lm  -fno-fast-math -fno-associative-math"`
+    * GCC+Radeon - `"-foffload=amdgcn-amdhsa='-march=gfx906' -foffload=-lm  -fno-fast-math -fno-associative-math"`
+    * LLVM+NVIDIA - `"-fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_75"`
+    * ICC - `"-qnextgen -fiopenmp -fopenmp-targets=spir64"`
+    * CCE+NVIDIA - `"-fopenmp-targets=nvptx64 -Xopenmp-target -march=sm_60"`
+ * `OMP_ALLOW_HOST` - `BOOL(ON|OFF)`, enabled by default, set to false if the compiler is unable to support dynamic selection of host/target devices. If disabled, running the binary with `--no-target` emits an error.
+
+
 
 If parts of your toolchain are installed at different places, you'll have to specify it manually, for example:
 
     cmake3 -Bbuild -H.  \
-    -DCMAKE_C_COMPILER=/nfs/software/x86_64/gcc/9.1.0/bin/gcc \
-    -DCMAKE_CXX_COMPILER=/nfs/software/x86_64/gcc/9.1.0/bin/g++ \
+    -DCMAKE_C_COMPILER=gcc \
+    -DCMAKE_CXX_COMPILER=g++ \
     -DCMAKE_BUILD_TYPE=Release \
+    -DOMP_OFFLOAD_FLAGS="-foffload=nvptx-none -foffload=-lm  -fno-fast-math -fno-associative-math"
 
 Proceed with compiling:
 
-    cmake3 --build build --target clover_leaf --config Release -j $(nproc)
-
+    cmake3 --build build --target cloverleaf --config Release -j $(nproc)
+
+## Known issues
+
+ * ICC 2021.1 Beta 20200602 requires `-DOMP_ALLOW_HOST=OFF`
+
 
 ## Running
 
-The main `clover_leaf` executable takes a `clover.in` file as parameter and outputs `clover.out` at working directory.
+The main `cloverleaf` executable takes a `clover.in` file as parameter and outputs `clover.out` at working directory.
 
 For example, after successful compilation, at **project root**:
 
-    ./build/clover_leaf --file InputDecks/clover_bm16_short.in
+    ./build/cloverleaf --file InputDecks/clover_bm16_short.in
 
 See [Tested configurations](#tested-configurations) for tested platforms and drivers.  
 
@@ -58,7 +71,8 @@ For help, use the `-h` flag:
 Options:
   -h  --help               Print the message
       --list               List available devices
+      --no-target          Use OMP fallback
       --device <INDEX>     Select device at INDEX from output of --list
-      --input              Custom clover.in file (defaults to clover.in if unspecified)
+      --file               Custom clover.in file (defaults to clover.in if unspecified)
 ```