Merge branch 'master' into doxygen

Signed-off-by: Stephane Rigaud <[email protected]>
clEsperanto · Jan 4, 2024 · 3bdd92d · 3bdd92d
2 parents 9e8e430 + e1c0119
commit 3bdd92d
Show file tree

Hide file tree

Showing 295 changed files with 14,068 additions and 6,672 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -9,8 +9,3 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "weekly"
-
-  # - package-ecosystem: "gitsubmodule" # See documentation for possible values
-  #   directory: "/" # Location of package manifests
-  #   schedule:
-  #     interval: "weekly"
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -1,4 +1,4 @@
-name: Build & test
+name: Build
 
 on:
   push:
@@ -7,11 +7,8 @@ on:
     branches: [master]
 
 env:
-  clespernato-kernels-git-tag: "d2348d964b1eaff60c60d7b3e18a59c2c8e3a84e"
-  opencl-sdk-git-tag: "v2022.09.30"
-  opencl-icd-loader-git-tag: "v2022.09.30"
-  opencl-headers-git-tag: "v2022.09.30"
-  opencl-clhpp-git-tag: "v2022.09.30"
+  opencl-icd-loader-git-tag: "v2023.04.17"
+  opencl-headers-git-tag: "v2023.04.17"
 
 jobs:
   build:
@@ -45,7 +42,7 @@ jobs:
     steps:
 
       - name: Setup Python
-        uses: actions/setup-python@v4.6.1
+        uses: actions/setup-python@v5.0.0
         with:
           python-version: 3.8
 
@@ -56,66 +53,66 @@ jobs:
         shell: bash
 
       - name: Get specific version of CMake, Ninja
-        uses: lukka/get-cmake@v3.26.4
+        uses: lukka/get-cmake@v3.27.7
 
       - name: Install ocl-icd-loader - windows
         if: matrix.os == 'windows-latest'
         run: |
           cd ..
           git clone --branch ${{ env.opencl-icd-loader-git-tag }} https://github.com/KhronosGroup/OpenCL-ICD-Loader
           git clone --branch ${{ env.opencl-headers-git-tag }} https://github.com/KhronosGroup/OpenCL-Headers
-          cmake -D CMAKE_INSTALL_PREFIX="${{ github.workspace }}/OpenCL-Headers/install" -S "./OpenCL-Headers" -B "./OpenCL-Headers/build" 
+          # cmake -D CMAKE_INSTALL_PREFIX="./OpenCL-Headers/install" -S "./OpenCL-Headers" -B "./OpenCL-Headers/build" 
+          cmake -S "./OpenCL-Headers" -B "./OpenCL-Headers/build" 
           cmake --build "./OpenCL-Headers/build" --parallel 6 --target install
-          cmake -D CMAKE_PREFIX_PATH="${{ github.workspace }}/OpenCL-Headers/install" -D OPENCL_ICD_LOADER_HEADERS_DIR="./OpenCL-Headers/install/include" -S "./OpenCL-ICD-Loader" -B "./OpenCL-ICD-Loader/build" -A x64
+          cmake -D CMAKE_PREFIX_PATH="./OpenCL-Headers" -D OPENCL_ICD_LOADER_HEADERS_DIR="./OpenCL-Headers/" -S "./OpenCL-ICD-Loader" -B "./OpenCL-ICD-Loader/build" -A x64
           cmake --build "./OpenCL-ICD-Loader/build" --parallel 6 --target install --config Release
         shell: bash -l {0}
 
-      - name: Install cuda toolkit (windows)
-        if: matrix.os == 'windows-latest'
-        uses: Jimver/[email protected]
-        id: cuda-toolkit
-        with:
-          cuda: '12.1.0'
+      # - name: Install cuda toolkit (windows)
+      #   if: matrix.os == 'windows-latest'
+      #   uses: Jimver/[email protected]
+      #   id: cuda-toolkit
+      #   with:
+      #     cuda: '12.1.0'
 
       - name: Install ocl-icd-loader - linux
         if: matrix.os == 'ubuntu-latest'
         run: |
           cd ..
           git clone --branch ${{ env.opencl-icd-loader-git-tag }} https://github.com/KhronosGroup/OpenCL-ICD-Loader
           git clone --branch ${{ env.opencl-headers-git-tag }} https://github.com/KhronosGroup/OpenCL-Headers
-          cmake -D CMAKE_INSTALL_PREFIX="${{ github.workspace }}/OpenCL-Headers/install" -S "./OpenCL-Headers" -B "./OpenCL-Headers/build" 
-          cmake --build "./OpenCL-Headers/build" --parallel 6 --target install
-          cmake -D CMAKE_PREFIX_PATH="${{ github.workspace }}/OpenCL-Headers/install" -D OPENCL_ICD_LOADER_HEADERS_DIR="./OpenCL-Headers/install/include" -S "./OpenCL-ICD-Loader" -B "./OpenCL-ICD-Loader/build"
+          cmake -S "./OpenCL-Headers" -B "./OpenCL-Headers/build" 
+          sudo cmake --build "./OpenCL-Headers/build" --parallel 6 --target install
+          cmake -D CMAKE_PREFIX_PATH="./OpenCL-Headers/" -D OPENCL_ICD_LOADER_HEADERS_DIR="./OpenCL-Headers/" -S "./OpenCL-ICD-Loader" -B "./OpenCL-ICD-Loader/build"
           sudo cmake --build "./OpenCL-ICD-Loader/build" --parallel 6 --target install --config Release
         shell: bash -l {0}
 
-      - name: Install cuda toolkit (ubuntu)
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin
-          sudo mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600
-          wget https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda-repo-ubuntu2204-12-1-local_12.1.1-530.30.02-1_amd64.deb
-          sudo dpkg -i cuda-repo-ubuntu2204-12-1-local_12.1.1-530.30.02-1_amd64.deb
-          sudo cp /var/cuda-repo-ubuntu2204-12-1-local/cuda-*-keyring.gpg /usr/share/keyrings/
-          sudo apt-get update
-          sudo apt-get -y install cuda
+      # - name: Install cuda toolkit (ubuntu)
+      #   if: matrix.os == 'ubuntu-latest'
+      #   run: |
+      #     wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin
+      #     sudo mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600
+      #     wget https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda-repo-ubuntu2204-12-1-local_12.1.1-530.30.02-1_amd64.deb
+      #     sudo dpkg -i cuda-repo-ubuntu2204-12-1-local_12.1.1-530.30.02-1_amd64.deb
+      #     sudo cp /var/cuda-repo-ubuntu2204-12-1-local/cuda-*-keyring.gpg /usr/share/keyrings/
+      #     sudo apt-get update
+      #     sudo apt-get -y install cuda
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
       - name: compile CLIc - linux
         if: matrix.os == 'ubuntu-latest'
         run : |
-          pwd
-          cmake -S "${{ github.workspace }}" -B "${{ runner.workspace }}/build" -DCMAKE_BUILD_TYPE=${{ matrix.cmake-build-type }} -DBUILD_DOCUMENTATION=OFF -DBUILD_BENCHMARK=OFF -DOpenCL_LIBRARIES:FILEPATH="/usr/local/lib/libOpenCL.so" -DOpenCL_INCLUDE_DIRS:PATH="${{ github.workspace }}/OpenCL-Headers/install/include/"
+          cmake -S "${{ github.workspace }}" -B "${{ runner.workspace }}/build" -DCMAKE_BUILD_TYPE=${{ matrix.cmake-build-type }} -DBUILD_DOCUMENTATION=OFF -DBUILD_BENCHMARK=OFF -DOpenCL_LIBRARIES:FILEPATH="/usr/local/lib/libOpenCL.so" -DOpenCL_INCLUDE_DIRS:PATH="/usr/local/include/"
           cmake --build "${{ runner.workspace }}/build" --parallel 10 --config Release
         shell: bash -l {0}    
 
       - name: compile CLIc - windows
         if: matrix.os == 'windows-latest'
         run : |
-          cmake -S "${{ github.workspace }}" -B "${{ runner.workspace }}/build" -DCMAKE_BUILD_TYPE=${{ matrix.cmake-build-type }} -DBUILD_DOCUMENTATION=OFF -DBUILD_BENCHMARK=OFF -DOpenCL_LIBRARIES:FILEPATH="C:/Program Files/OpenCL-ICD-Loader/lib/OpenCL.lib" -DOpenCL_INCLUDE_DIRS:PATH="${{ github.workspace }}/OpenCL-Headers/install/include/"
+          cmake -S "${{ github.workspace }}" -B "${{ runner.workspace }}/build" -DCMAKE_BUILD_TYPE=${{ matrix.cmake-build-type }} -DBUILD_DOCUMENTATION=OFF -DBUILD_BENCHMARK=OFF -DOpenCL_LIBRARIES:FILEPATH="C:/Program Files/OpenCL-ICD-Loader/lib/OpenCL.lib" -DOpenCL_INCLUDE_DIRS:PATH="C:/Program Files (x86)/OpenCLHeaders/include/"
           cmake --build "${{ runner.workspace }}/build" --parallel 10 --config Release
         shell: bash -l {0}
 

diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml
@@ -1,4 +1,4 @@
-name: clang-format Check
+name: Clang-format
 on: [pull_request]
 jobs:
   formatting-check:
@@ -7,7 +7,7 @@ jobs:
 
     steps:
       - name: git checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run clang-format style check for C/C++ programs.
         uses: jidicula/[email protected]

diff --git a/.github/workflows/tests_and_coverage.yml b/.github/workflows/tests_and_coverage.yml
@@ -1,4 +1,4 @@
-name: CI-Tests
+name: Tests
 
 on:
   push:
@@ -8,7 +8,7 @@ on:
 
 jobs:
   build:
-    name: ${{ matrix.os }}-${{ matrix.buildtype }}-Test-and-Coverage
+    name: ${{ matrix.os }}-${{ matrix.buildtype }}
     runs-on: ${{ matrix.os }}
     strategy:
       max-parallel: 1
@@ -17,14 +17,11 @@ jobs:
         os: [macos-latest]
         cxx: [clang++]
         cc: [clang]
-        triplet: [x64-osx]
         packages: [lcov gcovr]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
+        uses: actions/checkout@v4
 
       - name: Install packages
         shell: bash
@@ -46,19 +43,14 @@ jobs:
         run: |
           cd ${{ runner.workspace }}/build && ctest -C ${{ matrix.buildtype }} --output-on-failure
 
-      # - name: Format coverage
-      #   shell: bash
-      #   run: |
-      #     cd ${{ runner.workspace }}/build
-      #     lcov --directory . --capture --output-file coverage.info
-      #     lcov --remove coverage.info '/usr/*' "${HOME}"'/.cache/*' --output-file coverage.info
-      #     lcov --list coverage.info
-
       - name: Format coverage
         shell: bash
         run: |
           cd ${{ runner.workspace }}/build
-          gcovr --root .. --output coverage.xml
+          gcovr -r . --filter='.*/clic/src/.*' --exclude-throw-branches --exclude-unreachable-branches --xml -o coverage.xml
+          # lcov --base-directory . --directory . --capture --output-file coverage.info
+          # lcov --remove coverage.info '/usr/*' "${HOME}"'/.cache/*' ''*/deps_/*' --output-file coverage.info
+          # lcov --list coverage.info
 
       - name: Upload coverage report
         uses: codecov/codecov-action@v3

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/pocc/pre-commit-hooks
+    rev: v1.1.0
+    hooks:
+      - id: clang-format
+
+
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -12,6 +12,9 @@ include(${PROJECT_SOURCE_DIR}/cmake/CMakeDependencies.cmake)
 # managed third party libraries
 add_subdirectory(thirdparty)
 
+# include eigen library but exclude it from build
+include_directories(${EIGEN_SOURCE_DIR} EXCLUDE_FROM_ALL)
+
 # manage configuration files
 configure_file(
     "${CMAKE_CURRENT_SOURCE_DIR}/${LIBRARY_NAME_LOWERCASE}/${LIBRARY_NAME_LOWERCASE}.hpp.in"
@@ -76,6 +79,3 @@ endif(BUILD_TESTING)
 if (BUILD_DOCUMENTATION)
     include(${PROJECT_SOURCE_DIR}/cmake/CMakeDoxygen.cmake)
 endif()
-
-
-
diff --git a/README.md b/README.md
@@ -87,4 +87,9 @@ More information on how to add new kernels to the library can be found in the [d
 # __Feedback welcome!__
 clEsperanto is developed in the open because we believe in the [open source community](https://clij.github.io/clij2-docs/community_guidelines). 
 Feel free to drop feedback as [github issue](https://github.com/clEsperanto/CLIc_prototype/issues) or via [image.sc](https://image.sc).  
-And if you liked our work, star the repository, share it with your friends, and use it to make cool stuff!
+And if you liked our work, star the repository, share it with your friends, and use it to make cool stuff!
+
+## Acknowledgements
+
+We acknowledge support by the Deutsche Forschungsgemeinschaft under Germany’s Excellence Strategy (EXC2068) Cluster of Excellence Physics of Life of TU Dresden.
+This project has been made possible in part by grant number 2021-237734 ([GPU-accelerating Fiji and friends using distributed CLIJ, NEUBIAS-style, EOSS4](https://chanzuckerberg.com/eoss/proposals/gpu-accelerating-fiji-and-friends-using-distributed-clij-neubias-style/)) from the Chan Zuckerberg Initiative DAF, an advised fund of the Silicon Valley Community Foundation.
diff --git a/clic/clic.hpp.in b/clic/clic.hpp.in
@@ -12,7 +12,7 @@
 #  ifdef __APPLE__
 #    include <OpenCL/opencl.h>
 #  else
-#    include <CL/cl.h>
+#    include <CL/opencl.h>
 #  endif
 #endif
 
@@ -23,4 +23,10 @@
 #  include <nvrtc.h>
 #endif
 
+#if USE_OPENCL
+  const constexpr size_t GPU_MEM_PTR_SIZE = sizeof(cl_mem);
+#else
+  const constexpr size_t GPU_MEM_PTR_SIZE = sizeof(void *);
+#endif
+
 #endif // __CLIC_HPP
diff --git a/clic/include/array.hpp b/clic/include/array.hpp
@@ -5,63 +5,83 @@
 #include "device.hpp"
 #include "utils.hpp"
 
-#include <algorithm>
-#include <variant>
-
 namespace cle
 {
 
-class Array : public std::enable_shared_from_this<Array>
+class Array
 {
 public:
   using Pointer = std::shared_ptr<Array>;
+
   static auto
   New() -> Array::Pointer
   {
     return std::shared_ptr<Array>(new Array());
   }
   static auto
-  create(const size_t &          width,
-         const size_t &          height,
-         const size_t &          depth,
+  create(size_t                  width,
+         size_t                  height,
+         size_t                  depth,
+         size_t                  dimension,
          const dType &           data_type,
          const mType &           mem_type,
          const Device::Pointer & device_ptr) -> Array::Pointer;
   static auto
-  create(const size_t &          width,
-         const size_t &          height,
-         const size_t &          depth,
+  create(size_t                  width,
+         size_t                  height,
+         size_t                  depth,
+         size_t                  dimension,
          const dType &           data_type,
          const mType &           mem_type,
          const void *            host_data,
          const Device::Pointer & device_ptr) -> Array::Pointer;
   static auto
-  create(Array::Pointer array) -> Array::Pointer;
+  create(const Array::Pointer & array) -> Array::Pointer;
 
   friend auto
   operator<<(std::ostream & out, const Array::Pointer & array) -> std::ostream &;
 
+
   auto
   allocate() -> void;
+
   auto
   write(const void * host_data) -> void;
+  auto
+  write(const void * host_data, const std::array<size_t, 3> & region, const std::array<size_t, 3> & buffer_origin)
+    -> void;
+  auto
+  write(const void * host_data, size_t x_coord, size_t y_coord, size_t z_coord) -> void;
+
   auto
   read(void * host_data) const -> void;
+  auto
+  read(void * host_data, const std::array<size_t, 3> & region, const std::array<size_t, 3> & buffer_origin) const
+    -> void;
+  auto
+  read(void * host_data, size_t x_coord, size_t y_coord, size_t z_coord) const -> void;
+
   auto
   copy(const Array::Pointer & dst) const -> void;
   auto
-  fill(const float & value) const -> void;
+  copy(const Array::Pointer &        dst,
+       const std::array<size_t, 3> & region,
+       const std::array<size_t, 3> & src_origin,
+       const std::array<size_t, 3> & dst_origin) const -> void;
+
+  auto
+  fill(float value) const -> void;
 
   [[nodiscard]] auto
-  nbElements() const -> size_t;
+  size() const -> size_t;
   [[nodiscard]] auto
   width() const -> size_t;
   [[nodiscard]] auto
   height() const -> size_t;
   [[nodiscard]] auto
   depth() const -> size_t;
   [[nodiscard]] auto
-  bytesPerElements() const -> size_t;
+  itemSize() const -> size_t;
   [[nodiscard]] auto
   dtype() const -> dType;
   [[nodiscard]] auto
@@ -71,6 +91,8 @@ class Array : public std::enable_shared_from_this<Array>
   [[nodiscard]] auto
   dim() const -> unsigned int;
   [[nodiscard]] auto
+  dimension() const -> unsigned int;
+  [[nodiscard]] auto
   initialized() const -> bool;
   [[nodiscard]] auto
   shortType() const -> std::string;
@@ -86,13 +108,15 @@ class Array : public std::enable_shared_from_this<Array>
   using MemoryPointer = std::shared_ptr<void *>;
 
   Array() = default;
-  Array(const size_t &          width,
-        const size_t &          height,
-        const size_t &          depth,
+  Array(size_t                  width,
+        size_t                  height,
+        size_t                  depth,
+        size_t                  dimension,
         const dType &           data_type,
         const mType &           mem_type,
         const Device::Pointer & device_ptr);
 
+  size_t          dim_ = 1;
   size_t          width_ = 1;
   size_t          height_ = 1;
   size_t          depth_ = 1;